def build():
########sizes
mergins = 20
win_size = (800, 700)
top_size = (win_size[0] - 2 * mergins, 30)
side_size = (100, win_size[1] - top_size[1] - 2 * mergins)
word_size = (win_size[0] - mergins * 2 - side_size[0], 80)
#####coordinates
top_cord = (mergins, mergins)
side_cord = (mergins, mergins + top_size[1])
word_cord = (side_size[0] + mergins, mergins + top_size[1])
#####initializing
pygame.init()
screen = pygame.display.set_mode(win_size)
#####
buttons = [
Button(text="Quit"),
Button(text="Setup"),
Button(text="Depth")
]
side_bar = Sidebar(side_cord,
w=side_size[0],
h=side_size[1],
buttons=buttons)
#####
top_bar = topbar.topBar(top_size, pos=top_cord)
#####
word_bar = topbar.word(word_size, "word", pos=word_cord)
clock = pygame.time.Clock()
#####
exit = False
while not exit:
e = pygame.event.wait()
if e.type == pygame.QUIT:
exit = True
cntnr = pygame.Surface(
(win_size[0] - 2 * mergins + 4, win_size[1] - 2 * mergins + 4))
####
import random
r = random.randint(23, 150)
g = random.randint(23, 150)
b = random.randint(23, 150)
####
cntnr.fill((r, g, b))
side_surf = side_bar.draw_buttons()
screen.blit(cntnr, (mergins - 2, mergins - 2))
screen.blit(top_bar, top_cord)
screen.blit(side_surf, side_cord)
screen.blit(word_bar, word_cord)
pygame.display.update()
clock.tick(40)
pygame.quit()
class PykinectInt:
DEPTH_WINSIZE = 320,240
VIDEO_WINSIZE = 640,480
###STATES
SETUP = 0;
RECORDING = 1;
FEEDBACK = 2;
READY = 3;
WAIT = 4;
###modes
USER = 0;
TRAINING = 1;
#### Limits
READY_COUNTER=3;
RECORDING_COUNTER=2;
FEEDBACK_COUNTER = 4;
WAIT_COUNTER=4;
WORDS = {"pause: ":"to pause","run: ": "to unpause","quit: ":"to quit","repeat: ": "to repeat the last word"}
SETUP_WORDS = {"train":"to start practice mode", "test": "To start test mode"}
def __init__(self,screen,backend = {}):
self.screen = screen;
self.screen_lock = thread.allocate()
self.draw_skeleton = True
self.video_display = False
self.dispInfo = pygame.display.Info()
self.skeleton_to_depth_image = nui.SkeletonEngine.skeleton_to_depth_image
self.control_words = ['pause','run','quit','repeat',"train",'test']
self.paused = False;
self.skeletons = None
self.DEPTH_WINSIZE = 320,240
self.VIDEO_WINSIZE = 640,480
self.skeletal_map = []
self.state = self.SETUP;
self.mode = self.TRAINING;
self.backend = backend;
self.wordlist = CircularArray(backend['words'])
self.word = "None"
self.test_word = self.wordlist.roll()
self.backend_wait = True;
self.show_depth = False;
####
if self.video_display:
size = self.dispInfo.current_w-self.VIDEO_WINSIZE[0];
else:
size = self.dispInfo.current_w-self.DEPTH_WINSIZE[0];
size = size + 100
#self.clock_image = resize((size,size), ou_img="ui/images/_clock.gif");
self.sent_data = False;
self.use_speech = True;
self.repeat = False
self.camera_surf = pygame.Surface(self.DEPTH_WINSIZE)
##########
self.counter = self.READY_COUNTER;
self.action = Text(self.screen,w=100, h=50,pos=(485,0),text=self.test_word[0],color=THECOLORS['white']);
self.count = Text(self.screen,w=100, h=100,pos=(485,55),text=str(self.counter),color=THECOLORS['white']);
####general state display paramters
self.mergin_side = 20;
self.mergin_top = 20;
###top bar
self.top_bar_size = (self.dispInfo.current_w-2*self.mergin_side,70);
self.topbar_pos = (self.mergin_side,self.mergin_side)
self.topbar = bars.topBar(self.top_bar_size,pos=self.topbar_pos);
###side bar
self.side_bar_w = 100;
self.side_bar_h = self.dispInfo.current_h-self.mergin_top*2-self.top_bar_size[1];
self.side_bar_pos = (self.mergin_side,self.top_bar_size[1]+self.mergin_top);
###word bar
w = self.dispInfo.current_w-self.side_bar_w-2*self.mergin_side
self.word_bar_size = (w,70);
self.word_bar_pos = (self.mergin_side+self.side_bar_w,self.mergin_side+self.top_bar_size[1])
###camera feedback pos
self.camera_feed_pos = (self.mergin_side+self.side_bar_w,self.word_bar_pos[1]+self.word_bar_size[1]);
####SETUP display parameters
self.train_button_pos = (100,100);
self.train_button = Button(pos=self.train_button_pos,text="TRAINING");
#++++++++++
self.user_button_pos = (100,210);
#++++++++
self.depth_button = Button(text="DEPTH")
#++++++
self.user_button = Button(pos=self.user_button_pos,text="USER");
self.setup_sidebar = Sidebar(self.side_bar_pos,w=self.side_bar_w,h=self.side_bar_h,buttons=[self.train_button,self.user_button,self.depth_button])
self.setup_sidebar_surf = self.setup_sidebar.draw_buttons()
###Text input
self.text_in_h = 40;
self.text_in_w = 100;
self.text_in_pos = (self.camera_feed_pos[0]+self.DEPTH_WINSIZE[0]+10,self.camera_feed_pos[1])
self.text_input = InputBox(self.text_in_pos[0], self.text_in_pos[1], self.text_in_w, self.text_in_h)
####READY display parameters
self.quit_button = Button(text="QUIT");
#++++++++++
self.setup_button = Button(text="SETUP");
#++++++
self.puase_button = Button(text="PAUSE");
#++++++
self.sidar_bar = Sidebar(self.side_bar_pos,w=self.side_bar_w,h=self.side_bar_h,buttons=[self.quit_button,self.puase_button,self.setup_button,self.depth_button])
self.sidebar_surf = self.sidar_bar.draw_buttons()
#++++++
#self.clock_pos = (self.camera_feed_pos[0]+self.DEPTH_WINSIZE[0]+10,self.camera_feed_pos[1]+self.text_in_h)
self.clock_pos = (self.camera_feed_pos[0]+self.DEPTH_WINSIZE[0]+70,self.camera_feed_pos[1])
self.clock = Clock(size=self.DEPTH_WINSIZE[1] + 30);
####RECODRING display parameters
####FEEDBACK parameters
self.feedback_bar_pos=(self.word_bar_pos[0], self.camera_feed_pos[1]+self.DEPTH_WINSIZE[1]+10);
self.feedback_bar_size = self.word_bar_size;
self.speech_thread = SpeechTrigger(self);
self.listen = False;
###
self.ctl_word_size = 40;
self.ctl_pose = self.camera_feed_pos[0],self.camera_feed_pos[1]+self.DEPTH_WINSIZE[1]+30
self.ctl_size = self.word_bar_size[0],300
self.clt_words=ControlWords(self.WORDS,font_size=self.ctl_word_size,pose=self.ctl_pose,size=self.ctl_size)
self.setup_clt_words=ControlWords(self.SETUP_WORDS,font_size=self.ctl_word_size,pose=self.ctl_pose,size=self.ctl_size)
self.ctl_surf = self.clt_words.show()
self.setup_ctl_surf=self.setup_clt_words.show()
### Feedback bars
self.congrats_bar = bars.congrats(self.feedback_bar_size,pos = self.feedback_bar_pos);
self.no_data_bar = bars.noData(self.feedback_bar_size,pos = self.feedback_bar_pos);
self.processing_bar = bars.processing(self.feedback_bar_size,pos = self.feedback_bar_pos)
self.gogo_bar = bars.gogo(size=(self.clock.size, self.clock.size),pos=self.clock_pos);
self.correct_word_pos = (self.feedback_bar_pos[0],(self.feedback_bar_pos[1]+self.feedback_bar_size[1]))
####Test words
self.train_bars = mappers.trainer_vocab_display_mapper(self);
self.test_bars = mappers.test__vocab_display_mapper(self);
self.sorry_bar_mapper = mappers.sorry_bar_mapper(self);
self.correct_word_bar = mappers.correct__word_display_mapper(self);
def surface_to_array(self,surface):
buffer_interface = surface.get_buffer()
address = ctypes.c_void_p()
size = Py_ssize_t()
_PyObject_AsWriteBuffer(buffer_interface,
ctypes.byref(address), ctypes.byref(size))
bytes = (ctypes.c_byte * size.value).from_address(address.value)
bytes.object = buffer_interface
return bytes
def pos_to_array(self,joint):
#print "joint", joint
return [joint.x,joint.y,joint.z]
def map_skeleton(self,skeleton):
skltl = Skeletal();
skltl.head = self.pos_to_array(skeleton.SkeletonPositions[JointId.Head]);
skltl.should_center = self.pos_to_array(skeleton.SkeletonPositions[JointId.ShoulderCenter]);
skltl.shoulder_left = self.pos_to_array(skeleton.SkeletonPositions[JointId.ShoulderLeft]);
skltl.shoulder_right = self.pos_to_array(skeleton.SkeletonPositions[JointId.ShoulderRight]);
skltl.elbow_left = self.pos_to_array(skeleton.SkeletonPositions[JointId.ElbowLeft]);
skltl.elbow_right = self.pos_to_array(skeleton.SkeletonPositions[JointId.ElbowRight]);
skltl.wrist_left = self.pos_to_array(skeleton.SkeletonPositions[JointId.WristLeft]);
skltl.wrist_right = self.pos_to_array(skeleton.SkeletonPositions[JointId.WristRight]);
skltl.hand_left =self.pos_to_array(skeleton.SkeletonPositions[JointId.HandLeft]);
skltl.hand_right =self.pos_to_array(skeleton.SkeletonPositions[JointId.HandRight]);
skltl.hip_center =self.pos_to_array(skeleton.SkeletonPositions[JointId.HipCenter]);
skltl.hip_left =self.pos_to_array(skeleton.SkeletonPositions[JointId.HipLeft]);
skltl.hip_right =self.pos_to_array(skeleton.SkeletonPositions[JointId.HandRight]);
skltl.ankle_left =self.pos_to_array(skeleton.SkeletonPositions[JointId.AnkleLeft]);
skltl.ankle_right =self.pos_to_array(skeleton.SkeletonPositions[JointId.AnkleRight]);
skltl.foot_left =self.pos_to_array(skeleton.SkeletonPositions[JointId.FootLeft]);
skltl.foot_right =self.pos_to_array(skeleton.SkeletonPositions[JointId.FootRight]);
skltl.knee_left =self.pos_to_array(skeleton.SkeletonPositions[JointId.KneeLeft]);
skltl.knee_right =self.pos_to_array(skeleton.SkeletonPositions[JointId.KneeRight]);
skltl.spine =self.pos_to_array(skeleton.SkeletonPositions[JointId.spine]);
return skltl;
def collect(self,skltns):
sf = [];
for index, sklton in enumerate(skltns):
sk = self.map_skeleton(sklton)
if sk.is_not_empty():
sf.append(sk);
if not sf==[]:
self.skeletal_map.append(ScanFrame(sf));
def draw_skeleton_data(self,pSkelton, index, positions, width = 4):
start = pSkelton.SkeletonPositions[positions[0]]
for position in itertools.islice(positions, 1, None):
next = pSkelton.SkeletonPositions[position.value]
if self.video_display:
curstart = self.skeleton_to_depth_image(start, self.VIDEO_WINSIZE[0], self.VIDEO_WINSIZE[1])
curend = self.skeleton_to_depth_image(next, self.VIDEO_WINSIZE[0], self.VIDEO_WINSIZE[1])
if curstart[0]<self.VIDEO_WINSIZE[0] and curstart[1]<self.VIDEO_WINSIZE[1]:
if curend[0]<self.VIDEO_WINSIZE[0] and curend[1]<self.VIDEO_WINSIZE[1]:
# curstart = curstart[0]+self.camera_feed_pos[0],curstart[1]+self.camera_feed_pos[1];
# curend = curend[0]+self.camera_feed_pos[0],curend[1]+self.camera_feed_pos[1]
pygame.draw.line(self.camera_surf, SKELETON_COLORS[index], curstart, curend, width);
else:
curstart = self.skeleton_to_depth_image(start, self.DEPTH_WINSIZE[0], self.DEPTH_WINSIZE[1])
curend = self.skeleton_to_depth_image(next, self.DEPTH_WINSIZE[0], self.DEPTH_WINSIZE[1])
if curstart[0]<self.DEPTH_WINSIZE[0] and curstart[1]<self.DEPTH_WINSIZE[1]:
if curend[0]<self.DEPTH_WINSIZE[0] and curend[1]<self.DEPTH_WINSIZE[1]:
# curstart = curstart[0]+self.camera_feed_pos[0],curstart[1]+self.camera_feed_pos[1];
# curend = curend[0]+self.camera_feed_pos[0],curend[1]+self.camera_feed_pos[1]
pygame.draw.line(self.camera_surf, SKELETON_COLORS[index], curstart, curend, width);
start = next
def draw_skeletons(self,skeletons):
if not self.show_depth:
self.camera_surf.fill((0,0,0))
for index, data in enumerate(skeletons):
# draw the Head
if self.video_display:
HeadPos = self.skeleton_to_depth_image(data.SkeletonPositions[JointId.Head], self.VIDEO_WINSIZE[0], self.VIDEO_WINSIZE[1])
else:
HeadPos = self.skeleton_to_depth_image(data.SkeletonPositions[JointId.Head], self.DEPTH_WINSIZE[0], self.DEPTH_WINSIZE[1])
self.draw_skeleton_data(data, index, SPINE, 10)
pygame.draw.circle(self.camera_surf, SKELETON_COLORS[index], (int(HeadPos[0]), int(HeadPos[1])), 20, 0)
# drawing the limbs
self.draw_skeleton_data(data, index, LEFT_ARM)
self.draw_skeleton_data(data, index, RIGHT_ARM)
self.draw_skeleton_data(data, index, LEFT_LEG)
self.draw_skeleton_data(data, index, RIGHT_LEG)
self.screen.blit(self.camera_surf,self.camera_feed_pos)
pygame.display.update()
def word_trigger(self,_words):
pygame.event.post(pygame.event.Event(SPEECHEVENT,words = _words));
def depth_frame_ready(self,frame):
if self.video_display:
return
#print "Adding depth........"
depth_surface = pygame.Surface(self.DEPTH_WINSIZE);
with self.screen_lock:
if self.show_depth:
address = self.surface_to_array(self.camera_surf)
frame.image.copy_bits(address)
del address
self.screen.blit(self.camera_surf,self.camera_feed_pos)
if self.skeletons is not None and self.draw_skeleton:
self.draw_skeletons(self.skeletons)
if self.state==self.RECORDING:
self.collect(self.skeletons);
disp(self)
pygame.display.update()
#print "deleted!"
def video_frame_ready(self,frame):
#print "video_display: ",self.video_display
if not self.video_display:
return
#print "Adding......."
vid_surface = pygame.Surface(self.VIDEO_WINSIZE);
with self.screen_lock:
if self.show_depth:
address = self.surface_to_array(self.camera_surf)
frame.image.copy_bits(address)
del address
self.screen.blit(self.camera_surf,self.camera_feed_pos);
if self.skeletons is not None and self.draw_skeleton:
self.draw_skeletons(self.skeletons)
if self.state==self.RECORDING:
self.collect(self.skeletons);
disp(self)
pygame.display.update()
def loop(self):
pygame.display.set_caption('Louder than words')
self.screen.fill(THECOLORS["black"])
if self.use_speech:
self.listen = True;
self.speech_thread.start()
kinect = nui.Runtime()
kinect.skeleton_engine.enabled = True
def post_frame(frame):
try:
pygame.event.post(pygame.event.Event(KINECTEVENT, skeletons = frame.SkeletonData))
except:
pass
kinect.skeleton_frame_ready += post_frame
kinect.depth_frame_ready += self.depth_frame_ready
kinect.video_frame_ready += self.video_frame_ready
kinect.video_stream.open(nui.ImageStreamType.Video, 2, nui.ImageResolution.Resolution640x480, nui.ImageType.Color)
kinect.depth_stream.open(nui.ImageStreamType.Depth, 2, nui.ImageResolution.Resolution320x240, nui.ImageType.Depth)
print('Controls: ')
print(' d - Switch to depth view')
print(' v - Switch to video view')
print(' s - Toggle displaing of the skeleton')
print(' u - Increase elevation angle')
print(' j - Decrease elevation angle')
pygame.time.set_timer(RECORDEVENT, 1000);
done = False
skeleton_counter = 0
clock = pygame.time.Clock()
while not done:
#r = random.randint(0,34);
#g = random.randint(0,34);
#b = random.randint(0,34);
#background_color = (r,g,b);
background_color = (1, 0, 79)
e = pygame.event.wait()
self.dispInfo = pygame.display.Info()
if e.type == pygame.QUIT:
self.listen = False
done = True
break
elif e.type == RECORDEVENT:
transition_handle(self,background_color,skeleton_counter)
elif e.type == KINECTEVENT:
skeletons = e.skeletons
#print "recieving skeletons"
###COLLECTING DATA
if self.state==self.RECORDING:
if not self.paused:
skeleton_counter+=1;
self.collect(skeletons);
if self.draw_skeleton:
#print "sending skeletons"
self.draw_skeletons(skeletons)
pygame.display.update()
elif e.type == KEYDOWN:
if e.key == K_ESCAPE:
done = True
break
elif e.key == K_d:
with self.screen_lock:
self.video_display = False
elif e.key == K_v:
with self.screen_lock:
self.screen = pygame.display.set_mode(self.VIDEO_WINSIZE,0,16)
self.video_display = True
elif e.key == K_s:
self.draw_skeleton = not self.draw_skeleton
elif e.key == K_u:
kinect.camera.elevation_angle = kinect.camera.elevation_angle + 2
elif e.key == K_j:
kinect.camera.elevation_angle = kinect.camera.elevation_angle - 2
elif e.key == K_x:
kinect.camera.elevation_angle = 2
######
##Click response
###
if e.type ==MOUSEBUTTONDOWN:
done=mouse_handle(self,done);
if e.type==SPEECHEVENT:
while len(e.words)!=0:
speech_word = e.words.pop(0)
done = word_handle(self,speech_word,done)
disp(self);
pygame.display.update();
clock.tick();
self.backend['save_data']()
pygame.quit()