def start_SLD_server(host_url, host_root, use_Edited_Model, i3d_models, LSTM_model, csvFile_dir, nTop= 3):
global id_and_prediction
global ids_fpaths_q
labels = VideoClasses(csvFile_dir)
rgb_model,oflow_model,lstmModel = load_Models(i3d_models, LSTM_model, use_Edited_Model)
threading.Thread(target=work, args=(host_url, 2)).start()
result = None
processed_vid = []
while True:
if not ids_fpaths_q.empty():
result = ids_fpaths_q.get()
if result is not None:
vid_id,vid_path = result
processed_vid.append(vid_id)
if not os.path.exists(host_root + vid_path):
raise ValueError("[ERROR]: Incorrect pathing to the videos - (check videos dirctories).")
vid = cv2.VideoCapture(host_root + vid_path)
rgbs,oflows,frames_count = vid2frames(vid)
results = preds(rgbs,oflows,frames_count,labels,use_Edited_Model,nTop,lstmModel,rgb_model,oflow_model,40,10,40)
if len(results) > 0:
predictions = Phase(results)
else:
predictions = [{'Unknown': 0.}, {'Unknown': 0.}, {'Unknown': 0.}]
print("My results:", results)
"""
if use_Edited_Model:
predictions,_ = i3d_LSTM_prediction(rgbs, oflows, labels, lstmModel, rgb_model, oflow_model, nTop=3)
else:
predictions,_ = get_predicts(rgbs, oflows, labels, oflow_model, rgb_model, nTop=3)
"""
#print("predictions:",predictions)
if not id_and_prediction_q.full() and vid_id in processed_vid:
id_and_prediction_q.put((vid_id,predictions))
processed_vid.remove(vid_id)
result = None
def start_SLD_server(host_url, host_root, i3d_models, csvFile_dir, nTop=3):
global id_and_prediction
global ids_fpaths_q
labels = VideoClasses(csvFile_dir)
rgb_model = None
oflow_model = None
if i3d_models["rgb"] is not None:
rgb_model = keras.models.load_model(i3d_models["rgb"])
if i3d_models["oflow"] is not None:
oflow_model = keras.models.load_model(i3d_models["oflow"])
#pool = multiprocessing.Pool(processes_num)
#m = multiprocessing.Manager()
#ids_fpaths_q = m.Queue()
#id_and_prediction_q = m.Queue()
#pool.apply_async(work, (host_url, ids_fpaths_q, id_and_prediction_q, 2))
threading.Thread(target=get_vid_dirs, args=(host_url, )).start()
result = None
while True:
result = currentObj
if result is not None:
vid_id, vid_path = result
if not os.path.exists(host_root + vid_path):
raise ValueError(
"[ERROR]: Incorrect pathing to the videos - (check videos dirctories)."
)
vid = cv2.VideoCapture(host_root + vid_path)
rgbs, oflows = vid2frames(vid)
predictions, _ = get_predicts(rgbs,
oflows,
labels,
oflow_model,
rgb_model,
nTop=3)
#print("predictions:",predictions)
currentResultObj = (vid_id, predictions)
def livedemo():
# dataset
diVideoSet = {
"sName": "chalearn",
"nClasses": 20, # number of classes
"nFramesNorm": 40, # number of frames per video
"nMinDim": 240, # smaller dimension of saved video-frames
"tuShape": (240, 320), # height, width
"nFpsAvg": 10,
"nFramesAvg": 50,
"fDurationAvg": 5.0
} # seconds
# files
sClassFile = "data-set/%s/%03d/class.csv" % (diVideoSet["sName"],
diVideoSet["nClasses"])
sVideoDir = "data-set/%s/%03d" % (diVideoSet["sName"],
diVideoSet["nClasses"])
print("\nStarting gesture recognition live demo ... ")
print(os.getcwd())
print(diVideoSet)
# load label description
oClasses = VideoClasses(sClassFile)
sModelFile = "model/20180627-0729-chalearn020-oflow-i3d-entire-best.h5"
h, w = 224, 224
keI3D = I3D_load(sModelFile, diVideoSet["nFramesNorm"], (h, w, 2),
oClasses.nClasses)
# open a pointer to the webcam video stream
oStream = video_start(device=1,
tuResolution=(320, 240),
nFramePerSecond=diVideoSet["nFpsAvg"])
#liVideosDebug = glob.glob(sVideoDir + "/train/*/*.*")
nCount = 0
sResults = ""
timer = Timer()
# loop over action states
while True:
# show live video and wait for key stroke
key = video_show(oStream,
"green",
"Press <blank> to start",
sResults,
tuRectangle=(h, w))
# start!
if key == ord(' '):
# countdown n sec
video_show(oStream,
"orange",
"Recording starts in ",
tuRectangle=(h, w),
nCountdown=3)
# record video for n sec
fElapsed, arFrames, _ = video_capture(oStream, "red", "Recording ", \
tuRectangle = (h, w), nTimeDuration = int(diVideoSet["fDurationAvg"]), bOpticalFlow = False)
print("\nCaptured video: %.1f sec, %s, %.1f fps" % \
(fElapsed, str(arFrames.shape), len(arFrames)/fElapsed))
# show orange wait box
frame_show(oStream,
"orange",
"Translating sign ...",
tuRectangle=(h, w))
# crop and downsample frames
arFrames = images_crop(arFrames, h, w)
arFrames = frames_downsample(arFrames, diVideoSet["nFramesNorm"])
# Translate frames to flows - these are already scaled between [-1.0, 1.0]
print("Calculate optical flow on %d frames ..." % len(arFrames))
timer.start()
arFlows = frames2flows(arFrames, bThirdChannel=False, bShow=True)
print("Optical flow per frame: %.3f" %
(timer.stop() / len(arFrames)))
# predict video from flows
print("Predict video with %s ..." % (keI3D.name))
arX = np.expand_dims(arFlows, axis=0)
arProbas = keI3D.predict(arX, verbose=1)[0]
nLabel, sLabel, fProba = probability2label(arProbas,
oClasses,
nTop=3)
sResults = "Sign: %s (%.0f%%)" % (sLabel, fProba * 100.)
print(sResults)
nCount += 1
# quit
elif key == ord('q'):
break
# do a bit of cleanup
oStream.release()
cv2.destroyAllWindows()
return
def train_I3D_oflow_end2end():
"""
Training the keras model.
:return: None
"""
sClassFile = "class.csv"
sOflowDir = "Training_data"
sModelDir = "model"
diTrainTop = {"fLearn": 1e-3, "nEpochs": 5}
diTrainAll = {"fLearn": 1e-4, "nEpochs": 1}
nBatchSize = 4
print("\nStarting I3D end2end training ...")
print(os.getcwd())
oClasses = VideoClasses(sClassFile)
# Load training data
path = os.path.join(sOflowDir, "train")
genFramesTrain = FramesGenerator(path, nBatchSize, 40, 224, 224, 2,
oClasses.liClasses)
path = os.path.join(sOflowDir, "val")
genFramesVal = FramesGenerator(path, nBatchSize, 40, 224, 224, 2,
oClasses.liClasses)
if (genFramesTrain):
print("Generated training data.")
if (genFramesVal):
print("Generated validation data")
# Load pretrained i3d model and adjust top layer
print("Load pretrained I3D flow model ...")
keI3DOflow = Inception_Inflated3d(include_top=False,
weights='flow_imagenet_and_kinetics',
input_shape=(40, 224, 224, 2))
print("Add top layers with %d output classes ..." % oClasses.nClasses)
keI3DOflow = layers_freeze(keI3DOflow)
keI3DOflow = add_i3d_top(keI3DOflow, oClasses.nClasses, dropout_prob=0.5)
if (keI3DOflow):
print("Model loaded successfully")
sLog = time.strftime("%Y%m%d-%H%M",
time.gmtime()) + "-%s%03d-oflow-i3d" % ("ISL", 105)
# Save the model
os.makedirs(sModelDir, exist_ok=True)
cpTopLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-top-last.h5",
verbose=1,
save_best_only=False,
save_weights_only=False)
cpTopBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-top-best.h5",
verbose=1,
save_best_only=True,
save_weights_only=False)
cpAllLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-entire-last.h5",
verbose=1,
save_weights_only=False,
save_best_only=False)
cpAllBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-entire-best.h5",
verbose=1,
save_best_only=True,
save_weights_only=False)
cbTensorBoard = keras.callbacks.TensorBoard(log_dir="logs",
histogram_freq=1,
update_freq='batch',
write_graph=True,
write_images=True,
batch_size=32)
callbacks1 = [cpTopLast, cpTopBest, cbTensorBoard]
callbacks2 = [cpAllBest, cpAllLast, cbTensorBoard]
# Fit top layers
print("Fit I3D top layers with generator: %s" % (diTrainTop))
optimizer = keras.optimizers.Adam(lr=diTrainTop["fLearn"])
keI3DOflow.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(generator=genFramesTrain,
validation_data=genFramesVal,
epochs=diTrainTop["nEpochs"],
workers=4,
use_multiprocessing=False,
max_queue_size=8,
verbose=1,
callbacks=callbacks1)
"""
Fit entire I3D model
print("Finetune all I3D layers with generator: %s" % (diTrainAll))
keI3DOflow = layers_unfreeze(keI3DOflow)
optimizer = keras.optimizers.Adam(lr = diTrainAll["fLearn"])
keI3DOflow.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(
generator = genFramesTrain,
validation_data = genFramesVal,
epochs = diTrainAll["nEpochs"],
workers = 4,
use_multiprocessing = False,
max_queue_size = 8,
verbose = 1,
callbacks=callbacks2)
"""
return
def train_mobile_lstm(diVideoSet, bImage=True, bOflow=True):
# feature extractor
diFeature = {
"sName": "mobilenet",
"tuInputShape": (224, 224, 3),
"tuOutputShape": (1024, )
}
#diFeature = {"sName" : "inception",
# "tuInputShape" : (299, 299, 3),
# "nOutput" : 2048}
# directories
sFolder = "%03d-%d" % (diVideoSet["nClasses"], diVideoSet["nFramesNorm"])
sClassFile = "data-set/%s/%03d/class.csv" % (diVideoSet["sName"],
diVideoSet["nClasses"])
sVideoDir = "data-set/%s/%03d" % (diVideoSet["sName"],
diVideoSet["nClasses"])
sImageDir = "data-temp/%s/%s/image" % (diVideoSet["sName"], sFolder)
sImageFeatureDir = "data-temp/%s/%s/image-mobilenet" % (
diVideoSet["sName"], sFolder)
sOflowDir = "data-temp/%s/%s/oflow" % (diVideoSet["sName"], sFolder)
sOflowFeatureDir = "data-temp/%s/%s/oflow-mobilenet" % (
diVideoSet["sName"], sFolder)
sModelDir = "model"
print("\nStarting training with MobileNet + LSTM ...")
print(os.getcwd())
# read the classes
oClasses = VideoClasses(sClassFile)
# Image: Load LSTM and train it
if bImage:
sLogPath = "log/" + time.strftime("%Y%m%d-%H%M", time.gmtime()) + \
"-%s%03d-image-mobile-lstm.csv"%(diVideoSet["sName"], diVideoSet["nClasses"])
print("Image log: %s" % sLogPath)
keModelImage = lstm_build(diVideoSet["nFramesNorm"],
diFeature["tuOutputShape"][0],
oClasses.nClasses,
fDropout=0.5)
train_feature_generator(sImageFeatureDir,
sModelDir,
sLogPath,
keModelImage,
oClasses,
nBatchSize=16,
nEpoch=100,
fLearn=1e-4)
# Oflow: Load LSTM and train it
if bOflow:
sLogPath = "log/" + time.strftime("%Y%m%d-%H%M", time.gmtime()) + \
"-%s%03d-flow-mobile-lstm.csv"%(diVideoSet["sName"], diVideoSet["nClasses"])
print("Optical flow log: %s" % sLogPath)
keModelOflow = lstm_build(diVideoSet["nFramesNorm"],
diFeature["tuOutputShape"][0],
oClasses.nClasses,
fDropout=0.5)
train_feature_generator(sOflowFeatureDir,
sModelDir,
sLogPath,
keModelOflow,
oClasses,
nBatchSize=16,
nEpoch=100,
fLearn=1e-4)
return
def train_I3D_oflow_end2end():
"""
Training the keras model.
:return: None
"""
# directories
sClassFile = "class.csv"
sOflowDir = "Training_data"
sModelDir = "model"
diTrainTop = {
"fLearn": 1e-6,
"nEpochs": 5}
diTrainAll = {
"fLearn": 1e-4,
"nEpochs": 1}
nBatchSize = 4
print("\nStarting I3D end2end training ...")
print(os.getcwd())
# read the ChaLearn classes
oClasses = VideoClasses(sClassFile)
# Load training data
# print(oClasses.liClasses)
path = os.path.join(sOflowDir, "train")
genFramesTrain = FramesGenerator(path, nBatchSize, 40, 224, 224, 2, oClasses.liClasses)
path = os.path.join(sOflowDir, "val")
genFramesVal = FramesGenerator(path, nBatchSize, 40, 224, 224, 2, oClasses.liClasses)
if (genFramesTrain):
print("train true")
if (genFramesVal):
print("val true")
# Load pretrained i3d model and adjust top layer
print("Load pretrained I3D flow model ...")
keI3DOflow = models.load_model("model/20190320-2118-ISL105-oflow-i3d-top-best.h5")
if (keI3DOflow):
print("loaded successfully")
# print(keI3DOflow.summary())
# Prep logging
sLog = time.strftime("%Y%m%d-%H%M", time.gmtime()) + "-%s%03d-oflow-i3d" % ("ISL", 105)
# Helper: Save results
csv_logger = keras.callbacks.CSVLogger("log/" + sLog + "-acc.csv", append=True)
# Helper: Save the model
os.makedirs(sModelDir, exist_ok=True)
cpTopLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" + sLog + "-top-last.h5", verbose=1,
save_best_only=False, save_weights_only=False)
cpTopBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" + sLog + "-top-best.h5", verbose=1,
save_best_only=False, save_weights_only=False)
cpAllLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" + sLog + "-entire-last.h5", verbose=1,
save_weights_only=False, save_best_only=False)
cpAllBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" + sLog + "-entire-best.h5", verbose=1,
save_best_only=False, save_weights_only=False)
cbTensorBoard = keras.callbacks.TensorBoard(log_dir="logs", histogram_freq=1, update_freq='batch',
write_graph=True, write_images=True, batch_size=32)
callbacks1 = [cpTopLast, cpTopBest, cbTensorBoard]
#callbacks2 = [cpAllBest, cpAllLast, cbTensorBoard]
# Fit top layers
print("Fit I3D top layers with generator: %s" % (diTrainTop))
optimizer = keras.optimizers.Adam(lr=diTrainTop["fLearn"], decay=1e-6)
keI3DOflow.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(
generator=genFramesTrain,
validation_data=genFramesVal,
epochs=diTrainTop["nEpochs"],
workers=4,
use_multiprocessing=False,
max_queue_size=8,
verbose=1,
callbacks=callbacks1)
'''
# Fit entire I3D model
print("Finetune all I3D layers with generator: %s" % (diTrainAll))
keI3DOflow = layers_unfreeze(keI3DOflow)
optimizer = keras.optimizers.Adam(lr = diTrainAll["fLearn"])
keI3DOflow.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(
generator = genFramesTrain,
validation_data = genFramesVal,
epochs = diTrainAll["nEpochs"],
workers = 4,
use_multiprocessing = False,
max_queue_size = 8,
verbose = 1,
callbacks=callbacks2)
'''
return
def train_I3D_oflow_end2end(diVideoSet):
"""
* Loads pretrained I3D model,
* reads optical flow data generated from training videos,
* adjusts top-layers adequately for video data,
* trains only news top-layers,
* then fine-tunes entire neural network,
* saves logs and models to disc.
"""
# directories
sFolder = "%03d-%d" % (diVideoSet["nClasses"], diVideoSet["nFramesNorm"])
#sClassFile = "data-set/chalearn/249/class.csv"
sClassFile = "data-set/%s/%03d/class.csv" % (diVideoSet["sName"],
diVideoSet["nClasses"])
#sVideoDir = "data-set/%s/%03d"%(diVideoSet["sName"], diVideoSet["nClasses"])
#sImageDir = "data-temp/%s/%s/image"%(diVideoSet["sName"], sFolder)
#sImageFeatureDir = "data-temp/%s/%s/image-i3d"%(diVideoSet["sName"], sFolder)
#sOflowDir = "data-temp/chalearn/249-40/image"
sOflowDir = "data-temp/%s/%s/oflow" % (diVideoSet["sName"], sFolder)
#sOflowFeatureDir = "data-temp/%s/%s/oflow-i3d"%(diVideoSet["sName"], sFolder)
sModelDir = "model"
diTrainTop = {"fLearn": 1e-3, "nEpochs": 3}
diTrainAll = {"fLearn": 1e-4, "nEpochs": 17}
nBatchSize = 4
print("\nStarting I3D end2end training ...")
print(os.getcwd())
# read the ChaLearn classes
oClasses = VideoClasses(sClassFile)
# Load training data
genFramesTrain = FramesGenerator(sOflowDir + "/train", nBatchSize,
diVideoSet["nFramesNorm"], 224, 224, 2,
oClasses.liClasses)
genFramesVal = FramesGenerator(sOflowDir + "/valid", nBatchSize,
diVideoSet["nFramesNorm"], 224, 224, 2,
oClasses.liClasses)
# Load pretrained i3d model and adjust top layer
print("Load pretrained I3D flow model ...")
keI3DOflow = Inception_Inflated3d(include_top=False,
weights='flow_imagenet_and_kinetics',
input_shape=(diVideoSet["nFramesNorm"],
224, 224, 2))
print("Add top layers with %d output classes ..." % oClasses.nClasses)
keI3DOflow = layers_freeze(keI3DOflow)
keI3DOflow = add_i3d_top(keI3DOflow, oClasses.nClasses, dropout_prob=0.5)
# Prep logging
sLog = time.strftime("%Y%m%d-%H%M", time.gmtime()) + \
"-%s%03d-oflow-i3d"%(diVideoSet["sName"], diVideoSet["nClasses"])
# Helper: Save results
csv_logger = keras.callbacks.CSVLogger("log/" + sLog + "-acc.csv",
append=True)
# Helper: Save the model
os.makedirs(sModelDir, exist_ok=True)
cpTopLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-above-last.h5",
verbose=0)
cpTopBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-above-best.h5",
verbose=1,
save_best_only=True)
cpAllLast = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-entire-last.h5",
verbose=0)
cpAllBest = keras.callbacks.ModelCheckpoint(filepath=sModelDir + "/" +
sLog + "-entire-best.h5",
verbose=1,
save_best_only=True)
# Fit top layers
print("Fit I3D top layers with generator: %s" % (diTrainTop))
optimizer = keras.optimizers.Adam(lr=diTrainTop["fLearn"])
keI3DOflow.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(generator=genFramesTrain,
validation_data=genFramesVal,
epochs=diTrainTop["nEpochs"],
workers=4,
use_multiprocessing=True,
max_queue_size=8,
verbose=1,
callbacks=[csv_logger, cpTopLast, cpTopBest])
# Fit entire I3D model
print("Finetune all I3D layers with generator: %s" % (diTrainAll))
keI3DOflow = layers_unfreeze(keI3DOflow)
optimizer = keras.optimizers.Adam(lr=diTrainAll["fLearn"])
keI3DOflow.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
count_params(keI3DOflow)
keI3DOflow.fit_generator(generator=genFramesTrain,
validation_data=genFramesVal,
epochs=diTrainAll["nEpochs"],
workers=4,
use_multiprocessing=True,
max_queue_size=8,
verbose=1,
callbacks=[csv_logger, cpAllLast, cpAllBest])
return
def livedemo():
fDurationAvg = 3.0 # seconds
# files
sClassFile = "class_ISL.csv"
print("\nStarting gesture recognition live demo ... ")
# load label description
oClasses = VideoClasses(sClassFile)
sModelFile = "model/20190322-1841-ISL105-oflow-i3d-top-best.h5"
h, w = 224, 224
keI3D = I3D_load(sModelFile, 40, (h, w, 2), oClasses.nClasses)
if (keI3D):
print("Model loaded successfully")
# open a pointer to the webcam video stream
oStream = video_start(device=0,
tuResolution=(320, 240),
nFramePerSecond=10)
nCount = 0
sResults = ""
timer = Timer()
# loop over action states
while True:
# show live video and wait for key stroke
key = video_show(oStream,
"green",
"Press key to start",
sResults,
tuRectangle=(h, w))
# start!
if (key == ord('3') or key == ord('5')):
# countdown n sec
video_show(oStream,
"orange",
"Recording starts in ",
tuRectangle=(h, w),
nCountdown=3)
# record video for n sec
if key == ord('3'):
fDurationAvg = 3
fElapsed, arFrames, _ = video_capture(
oStream,
"red",
"Recording ",
tuRectangle=(h, w),
nTimeDuration=int(fDurationAvg),
bOpticalFlow=False)
else:
fDurationAvg = 5
fElapsed, arFrames, _ = video_capture(
oStream,
"red",
"Recording ",
tuRectangle=(h, w),
nTimeDuration=int(fDurationAvg),
bOpticalFlow=False)
print("\nCaptured video: %.1f sec, %s, %.1f fps" %
(fElapsed, str(arFrames.shape), len(arFrames) / fElapsed))
# show orange wait box
frame_show(oStream,
"orange",
"Translating sign ...",
tuRectangle=(h, w))
# crop and downsample frames
arFrames = images_crop(arFrames, h, w)
arFrames = frames_downsample(arFrames, 40)
# Translate frames to flows - these are already scaled between [-1.0, 1.0]
print("Calculate optical flow on %d frames ..." % len(arFrames))
timer.start()
arFlows = frames2flows(arFrames, bThirdChannel=False, bShow=True)
print("Optical flow per frame: %.3f" %
(timer.stop() / len(arFrames)))
# predict video from flows
print("Predict video with %s ..." % (keI3D.name))
arX = np.expand_dims(arFlows, axis=0)
arProbas = keI3D.predict(arX, verbose=1)[0]
nLabel, sLabel, fProba = probability2label(arProbas,
oClasses,
nTop=3)
sResults = "Sign: %s (%.0f%%)" % (sLabel, fProba * 100.)
print(sResults)
nCount += 1
# quit
elif key == ord('q'):
break
oStream.release()
cv2.destroyAllWindows()
return
def unittest_videocapture():
print("--> Loading Nerual Network Model ...")
i3d_model = keras.models.load_model( "./model/20181011-1229-chalearn249-oflow-i3d-entire-best.h5")
pool = ThreadPool(processes=1)
cv2.namedWindow("Video", cv2.WINDOW_NORMAL)
cv2.resizeWindow("Video", 320, 240)
cv2.namedWindow("Optical flow", cv2.WINDOW_NORMAL)
cv2.resizeWindow("Optical flow", 224, 224)
# open a pointer to the video stream
print("--> Opening the stream...")
oStream = video_start(device = "rtsp://192.168.1.45:8080/h264_ulaw.sdp", tuResolution = (320, 240), nFramePerSecond = 15)
#oStream.set(3, 320)
#oStream.set(4, 240)
#liFrames = []
# loop over action states
sResults = ""
while True:
# show live video and wait for key stroke
key = video_show(oStream, "green", "Press <blank> to start", sResults)
# start!
if key == ord(' '):
# countdown n sec
video_show(oStream, sColor = "orange", sUpper = "Recording starts in ", sLower = None,
tuRectangle = (224, 224), nCountdown = 3)
# record video for n sec
fElapsed, liFrames, liFlows = video_capture(oStream, "red", "Recording ", nTimeDuration=5, bOpticalFlow=True)
# show orange wait box
frame_show(oStream, "orange", "Translating sign ...")
#_thread.start_new_thread( predict, (liFlows, "./model/20181011-1058-chalearn249-oflow-i3d-entire-best.h5", VideoClasses("./my_classes.csv")) )
#async_result = pool.apply_async(predict, (liFlows, i3d_model, VideoClasses("./my_classes.csv")) )
#top = async_result.get()
# run NN to translate video to label
top = predict(liFlows, i3d_model, VideoClasses("./my_classes.csv"))
#time.sleep(3)
#predict(liFlows, "./model/20181011-1058-chalearn249-oflow-i3d-entire-best.h5", VideoClasses("./my_classes.csv"))
oStream = video_start(device = "rtsp://192.168.1.45:8080/h264_ulaw.sdp", tuResolution = (320, 240), nFramePerSecond = 15)
#sResults = "Video duration {:.1f} sec, {} frames recorded, {:.1f} fps". \
#format(fElapsed, len(liFrames), len(liFrames)/fElapsed)
sResults = f"label:{top['detail']}-{top['confidence']}"
#video info
print("Video duration {:.1f} sec, {} frames recorded, {:.1f} fps". \
format(fElapsed, len(liFrames), len(liFrames)/fElapsed))
# ready for next video
elif key == ord("+"):
fFPS *= 2.
print("Frame per second increased from %.1f to %.1f" % (oStream.get(cv2.CAP_PROP_FPS),fFPS))
oStream.set(cv2.CAP_PROP_FPS, fFPS)
elif key == ord("-"):
fFPS /= 2.
print("Frame per second decreased from %.1f to %.1f" % (oStream.get(cv2.CAP_PROP_FPS), fFPS))
oStream.set(cv2.CAP_PROP_FPS, fFPS)
# quit
elif key == ord('q'):
break
cv2.waitKey(1)
# do a bit of cleanup
oStream.release()
cv2.destroyAllWindows()
return
"nMinDim": 240, # smaller dimension of saved video-frames
"tuShape": (224, 226), # height, width
"nFpsAvg": 10,
"nFramesAvg": 50,
"fDurationAvg": 5.0
} # seconds
# files
sClassFile = "class.csv"
print("\nStarting gesture recognition live demo ... ")
print(os.getcwd())
print(diVideoSet)
# load label description
oClasses = VideoClasses(sClassFile)
sModelFile = "epochs_001-val_acc_0.980.hdf5"
h, w = 224, 224
keI3D = I3D_load(sModelFile, diVideoSet["nFramesNorm"], (h, w, 2),
oClasses.nClasses)
def live():
gameDisplay.blit(carImg, (0, 0))
# open a pointer to the webcam video stream
oStream = video_start(device=1,
tuResolution=(320, 240),
nFramePerSecond=diVideoSet["nFpsAvg"])
def unittest_videocapture():
i3d_models = {
"oflow":
"./model/35class/20181023-0930-chalearn035-oflow-i3d-entire-best_downloaded.h5", #"./model/20181011-1229-chalearn249-oflow-i3d-entire-best.h5",
"rgb":
"./model/35class/20181023-1505-chalearn035-rgb-i3d-entire-best_download.h5"
} #"./model/20181015-1456-chalearn249-rgb-i3d-entire-best.h5"}
csvFile_dir = "./35class.csv"
camera_config = "rtsp://192.168.1.27:8080/h264_ulaw.sdp"
print("--> Loading Nerual Network Models ...")
rgb_model = None
oflow_model = None
if i3d_models["rgb"] is not None:
rgb_model = keras.models.load_model(i3d_models["rgb"])
if i3d_models["oflow"] is not None:
oflow_model = keras.models.load_model(i3d_models["oflow"])
#pool = ThreadPool(processes=4)
cv2.namedWindow("Video", cv2.WINDOW_NORMAL)
cv2.resizeWindow("Video", 320, 240)
cv2.namedWindow("Translate", cv2.WINDOW_NORMAL)
cv2.resizeWindow("Translate", 450, 300)
cv2.namedWindow("Optical flow", cv2.WINDOW_NORMAL)
cv2.resizeWindow("Optical flow", 224, 224)
# open a pointer to the video stream
print("--> Opening the stream...")
oStream = video_start(device=camera_config,
tuResolution=(320, 240),
nFramePerSecond=15)
#oStream.set(3, 320)
#oStream.set(4, 240)
#liFrames = []
# show live video and wait for key stroke
interval = False
key = video_show(oStream, "green", "Press <blank> to start", "")
# loop over action states
sResults = ["cümle:"]
blackImg = np.zeros((512, 512, 3), np.uint8) #cv2.imread("./my_black.jpg")
top = None
#async_result = None
while True:
# start!
if True:
blackImg = print_on_black(blackImg, sResults)
cv2.imshow("Translate", blackImg)
#key = cv2.waitKey(1) & 0xFF
if key == 114:
key = video_show(oStream, "green", "Press <blank> to start",
"")
sResults = ["cümle:"]
blackImg = np.zeros((512, 512, 3), np.uint8)
blackImg = print_on_black(blackImg, sResults)
cv2.imshow("Translate", blackImg)
key = video_show(oStream,
sColor="orange",
sUpper="next word in ",
sLower=None,
tuRectangle=(224, 224),
nCountdown=3)
print("keyyyy:", key)
if key == 114:
continue
# countdown n sec
#video_show(oStream, sColor = "orange", sUpper = "Recording starts in ", sLower = None,
#tuRectangle = (224, 224), nCountdown = 3)
# record video for n sec
if key != 100:
fElapsed, liFrames, liFlows, key = video_capture(
oStream,
"red",
"Recording ",
nTimeDuration=5,
bOpticalFlow=True)
if key != 114 and key != 100:
#async_result = pool.apply_async(predict, (liFlows, i3d_model, VideoClasses("./my_classes.csv")) )
top = get_predicts(liFrames,
liFlows,
VideoClasses(csvFile_dir),
oflow_model,
rgb_model,
nTop=3)
sResults.append(f"{top['detail']}")
elif key == 100:
if len(sResults) > 1:
sResults.pop()
key = 0
blackImg = np.zeros((512, 512, 3), np.uint8)
blackImg = print_on_black(blackImg, sResults)
cv2.imshow("Translate", blackImg)
continue
# show orange wait box
frame_show(oStream, "orange", "Translating sign ...")
#_thread.start_new_thread( predict, (liFlows, "./model/20181011-1058-chalearn249-oflow-i3d-entire-best.h5", VideoClasses("./my_classes.csv")) )
key = 0
# run NN to translate video to label
#top = predict(liFlows, i3d_model, VideoClasses("./my_classes.csv"))
#time.sleep(3)
#predict(liFlows, "./model/20181011-1058-chalearn249-oflow-i3d-entire-best.h5", VideoClasses("./my_classes.csv"))
#rtsp://192.168.1.85:8554/live.sdp
oStream = video_start(device=camera_config,
tuResolution=(320, 240),
nFramePerSecond=15)
#sResults = "Video duration {:.1f} sec, {} frames recorded, {:.1f} fps". \
#format(fElapsed, len(liFrames), len(liFrames)/fElapsed)
#sResults = f"label:{top['detail']}-{top['confidence']}"
#video info
print("Video duration {:.1f} sec, {} frames recorded, {:.1f} fps". \
format(fElapsed, len(liFrames), len(liFrames)/fElapsed))
# ready for next video
elif key == ord("+"):
fFPS *= 2.
print("Frame per second increased from %.1f to %.1f" %
(oStream.get(cv2.CAP_PROP_FPS), fFPS))
oStream.set(cv2.CAP_PROP_FPS, fFPS)
elif key == ord("-"):
fFPS /= 2.
print("Frame per second decreased from %.1f to %.1f" %
(oStream.get(cv2.CAP_PROP_FPS), fFPS))
oStream.set(cv2.CAP_PROP_FPS, fFPS)
# quit
elif key == ord('q'):
break
cv2.waitKey(1)
# do a bit of cleanup
oStream.release()
cv2.destroyAllWindows()
return