def eval():
label_dict = get_gt_labels()
hand = "LH"
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
gt_list = []
pred_list = []
prob_dict = {}
tp = 0.0
fp = 0.0
for video in range(30001, 35878):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/depth/individual/train/%03d/K_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
most_common = 1
if depth_images is not None:
print folder, video, depth_images.shape,
probs = hand_classfier.classify(depth_images)
pred = np.argmax(probs, axis=1)
gt_list += [label_dict[video]] * len(pred)
pred_list += list(pred)
most_common, num_most_common = Counter(pred).most_common(1)[0]
print most_common, label_dict[video], np.max(probs[:, most_common])
# print pred, label_dict[video]
prob_dict[video] = [probs, frame_list]
if most_common == label_dict[video]:
tp += 1
else:
fp += 1
print len(gt_list), len(pred_list)
ftp = 0.0
for g, p in zip(gt_list, pred_list):
if g == p:
ftp += 1
print hand, ftp, len(gt_list), ftp / len(gt_list)
print hand, tp, tp + fp, tp / (tp + fp)
with open("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/predictions/depth/eval_%s.p" % hand, "wb") as fd:
cPickle.dump(prob_dict, fd)
def eval():
label_dict = get_gt_labels()
hand = "LH"
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
gt_list = []
pred_list = []
prob_dict = {}
tp = 0.0
fp = 0.0
for video in range(30001, 35878):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/rgb/individual/train/%03d/M_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
most_common = 1
if depth_images is not None:
print folder, video, depth_images.shape,
probs = hand_classfier.classify(depth_images)
pred = np.argmax(probs, axis=1)
gt_list += [label_dict[video]] * len(pred)
pred_list += list(pred)
most_common, num_most_common = Counter(pred).most_common(1)[0]
print most_common, label_dict[video], np.max(probs[:, most_common])
prob_dict[video] = [probs, frame_list]
if most_common == label_dict[video]:
tp += 1
else:
fp += 1
print len(gt_list), len(pred_list)
ftp = 0.0
for g, p in zip(gt_list, pred_list):
if g == p:
ftp += 1
print hand, ftp, len(gt_list), ftp / len(gt_list)
print hand, tp, tp + fp, tp / (tp + fp)
with open(
"/s/red/a/nobackup/cwc/skeleton/ChaLearn17/predictions/rgb/eval_%s.p"
% hand, "wb") as fd:
cPickle.dump(prob_dict, fd)
def __init__(self):
self.kinect = Kinect(SEGMENT_SIZE, ENGAGE_MIN, ENGAGE_MAX)
socket_started = False
self.socket_api = None
while (not socket_started):
try:
self.socket_api = SocketAPI(TCP_IP, TCP_PORT, BUFFER_SIZE,
ACK_TIMEOUT, TERMINATOR)
socket_started = True
except:
print("Socket didn't start, waiting 3 seconds...")
time.sleep(3)
print("Connected!")
self.HandModel = RealTimeHandRecognition("RH", 32, 2)
def features(hand):
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
for video in range(1, 5800):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/rgb/individual/valid/%03d/M_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
if depth_images is not None:
features = hand_classfier.features(depth_images)
print folder, video, depth_images.shape, features.shape
np.save("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/features/valid/%03d/K_%05d/%s_rgb_features.npy" % (
folder, video, hand), features)
np.save("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/features/valid/%03d/K_%05d/%s_rgb_frames.npy" % (
folder, video, hand), frame_list)
def valid(hand):
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
prob_dict = {}
for video in range(1, 5785):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/rgb/individual/valid/%03d/M_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
if depth_images is not None:
print folder, video, depth_images.shape
probs = hand_classfier.classify(depth_images)
prob_dict[video] = [probs, frame_list]
with open("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/predictions/rgb/valid_%s.p" % hand, "wb") as fd:
cPickle.dump(prob_dict, fd)
def features(hand):
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
for video in range(1, 5800):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/depth/individual/valid/%03d/K_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
if depth_images is not None:
features = hand_classfier.features(depth_images)
print folder, video, depth_images.shape, features.shape
np.save("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/features/valid/%03d/K_%05d/%s_depth_features.npy" % (
folder, video, hand), features)
np.save("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/features/valid/%03d/K_%05d/%s_depth_frames.npy" % (
folder, video, hand), frame_list)
def valid(hand):
num_gestures = 250
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
prob_dict = {}
for video in range(1, 5785):
folder = ((video - 1) / 200) + 1
dir_path = "/s/red/a/nobackup/cwc/skeleton/ChaLearn17/frames/hands/depth/individual/valid/%03d/K_%05d/" % (
folder, video)
depth_images, frame_list = get_all_frames(dir_path, hand)
if depth_images is not None:
print folder, video, depth_images.shape
probs = hand_classfier.classify(depth_images)
prob_dict[video] = [probs, frame_list]
with open("/s/red/a/nobackup/cwc/skeleton/ChaLearn17/predictions/depth/valid_%s.p" % hand, "wb") as fd:
cPickle.dump(prob_dict, fd)
class DepthClient:
def __init__(self):
self.kinect = Kinect(SEGMENT_SIZE, ENGAGE_MIN, ENGAGE_MAX)
socket_started = False
self.socket_api = None
while (not socket_started):
try:
self.socket_api = SocketAPI(TCP_IP, TCP_PORT, BUFFER_SIZE,
ACK_TIMEOUT, TERMINATOR)
socket_started = True
except:
print("Socket didn't start, waiting 3 seconds...")
time.sleep(3)
print("Connected!")
self.HandModel = RealTimeHandRecognition("RH", 32, 2)
def run(self):
while True:
frames = self.kinect.get()
if frames is None:
print("waiting for frames...")
time.sleep(1 / 30)
continue
(LH_probs, LH_out), (RH_probs, RH_out) = self.HandModel.classifyLR(
frames[0], frames[1])
LH_idx = np.argmax(LH_out)
LH_label = hand_postures[LH_idx]
RH_idx = np.argmax(RH_out)
RH_label = hand_postures[RH_idx]
#self.socket_api.send_to_server("user:hands:left:probs", LH_out)
#self.socket_api.set("user:hands:left:argmax", int(LH_idx))
self.socket_api.set("user:hands:left", LH_label)
#self.socket_api.send_to_server("user:hands:right:probs", RH_out)
#self.socket_api.set("user:hands:right:argmax", int(RH_idx))
self.socket_api.set("user:hands:right", RH_label)
print(LH_label, RH_label)
Experimental function to read each stream frame_hand from the server
"""
(frame_size, ) = struct.unpack("<i", recv_all(sock, 4))
return recv_all(sock, frame_size)
if __name__ == '__main__':
hand = sys.argv[1]
stream_id = streams.get_stream_id(hand)
if hand == "RH":
FRAME_TYPE = 1
elif hand == "LH":
FRAME_TYPE = 0
r = RealTimeHandRecognition(hand, 20)
gesture_list = os.listdir(
"/s/red/a/nobackup/cwc/hands/rgb_hands_new_frames/s_01")
gesture_list.sort()
fusion_map_dict = {
0: 1,
19: 2,
1: 3,
2: 4,
3: 8,
4: 10,
5: 11,
6: 13,
7: 14,
# By default read 100 frames
if __name__ == '__main__':
hand = sys.argv[1]
stream_id = streams.get_stream_id(hand)
gestures = list(
np.load(
"/s/red/a/nobackup/cwc/hands/real_time_training_data/%s/gesture_list.npy"
% hand))
gestures = [g.replace(".npy", "") for g in gestures]
num_gestures = len(gestures)
gestures += ['blind']
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
kinect_socket = connect('kinect', hand)
fusion_socket = connect('fusion', hand)
i = 0
hands_list = []
start_time = time.time()
while True:
try:
frame = recv_depth_frame(kinect_socket)
except KeyboardInterrupt:
break
except:
continue
# By default read 100 frames
if __name__ == '__main__':
sys.modules['RandomForest'] = RandomForest # To load the pickle
hand = sys.argv[1]
stream_id = streams.get_stream_id(hand)
gestures = list(np.load("/s/red/a/nobackup/cwc/hands/real_time_training_data/%s/gesture_list.npy" % hand))
gestures = [g.replace(".npy", "") for g in gestures]
num_gestures = len(gestures)
gestures += ['blind']
print hand, num_gestures
hand_classfier = RealTimeHandRecognition(hand, num_gestures)
# load random forest classifier
load_path = os.path.join('/s/red/a/nobackup/vision/jason/forest', '%s_forest.pickle' % hand)
print('Loading random forest checkpoint: %s' % load_path)
f = open(load_path, 'rb')
forest = pickle.load(f)
f.close()
print('%s forest loaded!' % hand)
# Connect to servers
kinect_socket_hand = connect('kinect', hand)
kinect_socket_speech = connect('kinect', 'Speech')
fusion_socket = connect('fusion', hand)
# the first speech frame usually has noise, ignore first frame speech
