def vid2frames(vid, oflow, pred_type, mul_oflow, oflow_pnum):
from frame import frames_downsample, images_rescale
from opticalflow import OpticalFlow, frames2flows
from oflow_multiprocessing import process_oflow
# Extract frames of a video and then normalize it to fixed-length
# Then make optical flow and RGB lists
# Input : video(Stream), RGB(Boolean), oflow(Boolean)
# Output : RGB-list, oflow-list
rgbFrames, oflowFrames = [], []
tuRectangle = (224, 224)
success, frame = vid.read()
frame_num = 0
while success:
if not success:
print("[warrning]: some video frames are corrupted.")
# see if this line effecting the results
frame = cv2.flip(frame, 1)
frame = cv2.resize(frame, tuRectangle, interpolation=cv2.INTER_AREA)
rgbFrames.append(frame)
success, frame = vid.read()
frame_num += 1
if len(rgbFrames) == 0:
raise ValueError("Could not extract webcam frames successfully.")
#rgbFrames = image_normalize(np.array(rgbFrames), 40)
if len(rgbFrames) < 40:
if oflow:
if mul_oflow:
oflowFrames = process_oflow(rgbFrames, oflow_pnum)
else:
oflowFrames = frames2flows(rgbFrames)
oflowFrames = frames_downsample(oflowFrames, 40)
rgbFrames = frames_downsample(np.array(rgbFrames), 40)
else:
if pred_type != "sentence":
rgbFrames = frames_downsample(np.array(rgbFrames), 40)
if oflow:
if mul_oflow:
oflowFrames = process_oflow(rgbFrames, oflow_pnum)
else:
oflowFrames = frames2flows(rgbFrames)
rgbFrames = images_rescale(rgbFrames)
return rgbFrames, oflowFrames, frame_num
def unittest_opticalflow_fromcamera():
timer = Timer()
# start video capture from webcam
oStream = video_start(1, (320, 240), 15)
# loop over action states
while True:
# show live video and wait for key stroke
key = video_show(oStream, "green", "Press <blank> to start", "")
# start!
if key == ord(' '):
# countdown n sec
video_show(oStream, "orange", "Recording starts in ", sLower = None, \
tuRectangle = (224, 224), nCountdown = 3)
# record video for n sec
fElapsed, arFrames, _ = video_capture(oStream, "red", "Recording ", \
tuRectangle = (224, 224), nTimeDuration = 5, bOpticalFlow = False)
print("\nCaptured video: %.1f sec, %s, %.1f fps" % \
(fElapsed, str(arFrames.shape), len(arFrames)/fElapsed))
# show orange wait box
frame_show(oStream, "orange", "Calculating optical flow ...")
# calculate and show optical flow
arFrames = images_crop(arFrames, 224, 224)
timer.start()
arFlows = frames2flows(arFrames, bThirdChannel=True)
print("Optical flow per frame: %.3f" %
(timer.stop() / len(arFrames)))
frames_show(flows2colorimages(arFlows),
int(5 * 1000 / len(arFrames)))
elif key == ord('f'):
unittest_fromfile()
# quit
elif key == ord('q'):
break
# do a bit of cleanup
oStream.release()
cv2.destroyAllWindows()
return
def livedemo():
# dataset
diVideoSet = {
"sName": "chalearn",
"nClasses": 20, # number of classes
"nFramesNorm": 40, # number of frames per video
"nMinDim": 240, # smaller dimension of saved video-frames
"tuShape": (240, 320), # height, width
"nFpsAvg": 10,
"nFramesAvg": 50,
"fDurationAvg": 5.0
} # seconds
# files
sClassFile = "data-set/%s/%03d/class.csv" % (diVideoSet["sName"],
diVideoSet["nClasses"])
sVideoDir = "data-set/%s/%03d" % (diVideoSet["sName"],
diVideoSet["nClasses"])
print("\nStarting gesture recognition live demo ... ")
print(os.getcwd())
print(diVideoSet)
# load label description
oClasses = VideoClasses(sClassFile)
sModelFile = "model/20180627-0729-chalearn020-oflow-i3d-entire-best.h5"
h, w = 224, 224
keI3D = I3D_load(sModelFile, diVideoSet["nFramesNorm"], (h, w, 2),
oClasses.nClasses)
# open a pointer to the webcam video stream
oStream = video_start(device=1,
tuResolution=(320, 240),
nFramePerSecond=diVideoSet["nFpsAvg"])
#liVideosDebug = glob.glob(sVideoDir + "/train/*/*.*")
nCount = 0
sResults = ""
timer = Timer()
# loop over action states
while True:
# show live video and wait for key stroke
key = video_show(oStream,
"green",
"Press <blank> to start",
sResults,
tuRectangle=(h, w))
# start!
if key == ord(' '):
# countdown n sec
video_show(oStream,
"orange",
"Recording starts in ",
tuRectangle=(h, w),
nCountdown=3)
# record video for n sec
fElapsed, arFrames, _ = video_capture(oStream, "red", "Recording ", \
tuRectangle = (h, w), nTimeDuration = int(diVideoSet["fDurationAvg"]), bOpticalFlow = False)
print("\nCaptured video: %.1f sec, %s, %.1f fps" % \
(fElapsed, str(arFrames.shape), len(arFrames)/fElapsed))
# show orange wait box
frame_show(oStream,
"orange",
"Translating sign ...",
tuRectangle=(h, w))
# crop and downsample frames
arFrames = images_crop(arFrames, h, w)
arFrames = frames_downsample(arFrames, diVideoSet["nFramesNorm"])
# Translate frames to flows - these are already scaled between [-1.0, 1.0]
print("Calculate optical flow on %d frames ..." % len(arFrames))
timer.start()
arFlows = frames2flows(arFrames, bThirdChannel=False, bShow=True)
print("Optical flow per frame: %.3f" %
(timer.stop() / len(arFrames)))
# predict video from flows
print("Predict video with %s ..." % (keI3D.name))
arX = np.expand_dims(arFlows, axis=0)
arProbas = keI3D.predict(arX, verbose=1)[0]
nLabel, sLabel, fProba = probability2label(arProbas,
oClasses,
nTop=3)
sResults = "Sign: %s (%.0f%%)" % (sLabel, fProba * 100.)
print(sResults)
nCount += 1
# quit
elif key == ord('q'):
break
# do a bit of cleanup
oStream.release()
cv2.destroyAllWindows()
return
def live():
gameDisplay.blit(carImg, (0, 0))
# open a pointer to the webcam video stream
oStream = video_start(device=1,
tuResolution=(320, 240),
nFramePerSecond=diVideoSet["nFpsAvg"])
timer = Timer()
sResults = ""
nCount = 0
while True:
# show live video and wait for key stroke
key = video_show(oStream,
"green",
"Press <blank> to start",
sResults,
tuRectangle=(h, w))
# start!
if key == ord(' '):
# countdown n sec
video_show(oStream,
"orange",
"Recording starts in ",
tuRectangle=(h, w),
nCountdown=3)
# record video for n sec
fElapsed, arFrames, _ = video_capture(oStream, "red", "Recording ", \
tuRectangle=(h, w), nTimeDuration=int(diVideoSet["fDurationAvg"]),
bOpticalFlow=False)
print("\nCaptured video: %.1f sec, %s, %.1f fps" % \
(fElapsed, str(arFrames.shape), len(arFrames) / fElapsed))
# show orange wait box
frame_show(oStream,
"orange",
"Translating sign ...",
tuRectangle=(h, w))
# crop and downsample frames
arFrames = images_crop(arFrames, h, w)
arFrames = frames_downsample(arFrames, diVideoSet["nFramesNorm"])
# Translate frames to flows - these are already scaled between [-1.0, 1.0]
print("Calculate optical flow on %d frames ..." % len(arFrames))
timer.start()
arFlows = frames2flows(arFrames, bThirdChannel=False, bShow=True)
print("Optical flow per frame: %.3f" %
(timer.stop() / len(arFrames)))
# predict video from flows
print("Predict video with %s ..." % (keI3D.name))
arX = np.expand_dims(arFlows, axis=0)
arProbas = keI3D.predict(arX, verbose=1)[0]
nLabel, sLabel, fProba = probability2label(arProbas,
oClasses,
nTop=3)
sResults = "Sign: %s (%.0f%%)" % (sLabel, fProba * 100.)
print(sResults)
nCount += 1
# quit
break
# do a bit of cleanup
message_display(sResults)
oStream.release()
cv2.destroyAllWindows()