def add_to_gesture(self, msg): """Add messages to ongoing gesture""" # start a new gesture if one is not ongoing if not self.g_ongoing: self.gesture = Gesture() self.g_start = time.time() self.g_ongoing = True # set sustain pedal control if msg.isController() and msg.getControllerNumber() == 64: if msg.getControllerValue() == 127: self.pedal = True else: self.pedal = False # mark note as sustained or completed if msg.isNoteOn(): self.sustained[msg.getNoteNumber()] = True if msg.isNoteOff(): self.sustained[msg.getNoteNumber()] = False # add midi message and time since start to gesture self.gesture.add_message(msg, time.time() - self.g_start) # record event time as last to inform gesture ending self.g_last = time.time() print_message(msg, self.in_name)
def FindHandFromTrack(self): # Get brightness from tracked hand kernel = np.ones((5, 5), np.float32) / 25 blurred = cv2.filter2D(self.currentFrame.copy(), -1, kernel) hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV) self.hand_lower_blue = self.AddValueToColorArray([ -config['hand']['hsv_hand_dec'][0], -config['hand']['hsv_hand_dec'][1], -config['hand']['hsv_hand_dec'][2] ], self.handPointHSV.copy()) self.hand_upper_blue = self.AddValueToColorArray([ config['hand']['hsv_hand_inc'][0], config['hand']['hsv_hand_inc'][1], config['hand']['hsv_hand_inc'][2] ], self.handPointHSV.copy()) mask = cv2.inRange(hsv, self.hand_lower_blue, self.hand_upper_blue) self.mask_rafined = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) search_hand_mask = self.mask_rafined.copy() search_hand = Gesture() if search_hand.InitFromMaskAndPosition( search_hand_mask, self.handTracked.centerX, self.handTracked.centerY) is False: self.foundHand = False else: self.timeSinceFoundHandTracked = time.time() self.handTracked = search_hand self.handPointHSV = hsv[self.handTracked.centerY][ self.handTracked.centerX] self.foundHand = True
def TryToTrackHand(self): lower_blue_brightness = 255 search_hand = Gesture() while lower_blue_brightness > 15: # define range of blue color in HSV lower_blue = np.array([config['hand']['hsv_lower_blue'][0], config['hand']['hsv_lower_blue'][1], lower_blue_brightness]) upper_blue = np.array([config['hand']['hsv_upper_blue'][0], config['hand']['hsv_upper_blue'][1], 255]) kernel = np.ones((5, 5), np.float32) / 25 blurred = cv2.filter2D(self.currentFrame.copy(), -1, kernel) hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV) # Threshold the HSV image to get only blue colors mask = cv2.inRange(hsv, lower_blue, upper_blue) self.mask_rafined = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) search_hand_mask = self.mask_rafined.copy() foundPalm = search_hand.SearchPalmFromMask(search_hand_mask) if foundPalm: # Set infos from tracked hand self.handTracked = search_hand self.timeSinceFoundHandTracked = time.time() self.handPointHSV = hsv[self.handTracked.centerY][self.handTracked.centerX] self.foundHand = True return lower_blue_brightness -= 10 self.foundHand = False
def fetch_pixels(): ''' Returns a tuple of pixels(inputs) and output labels ''' gestures = [] for user in user_number: path_to_csv = 'Dataset/user_' + str(user) + '/user_' + str( user) + '_loc.csv' print(path_to_csv) path_to_images = 'Dataset/user_' + str(user) + '/' list_of_images = os.listdir(path_to_images) list_of_images = filter(lambda x: '.csv' not in x, list_of_images) for image in list_of_images: im = Image.open(path_to_images + image) pix = im.load() rows, cols = im.size pixels = list() for row in range(rows): for col in range(cols): for p in pix[row, col]: pixels.append(p / 256) gesture = Gesture(image[0], pixels) gesture.generate_output() gestures.append(gesture) return gestures
def classifyGesture(self): minError = 2**31 - 1 # a large value minErrorIndex = -1 self.humanGesture = Gesture(self.gesturePoints, "Human Gesture") likelihoodScores = [0] * len(self.gestures) assessments = [{}] * len(self.gestures) for i in xrange(len(self.gestures)): assessments[i] = Gesture.compareGestures(self.gestures[i], self.humanGesture) errorList = [assessments[i][Gesture.totalError] \ for i in xrange(len(assessments))] index = errorList.index(min(errorList)) # Basic elimination to figure out if result is valid # show appropriate images in windows templateGestureRatio = max((self.gestures[index].distance /\ self.humanGesture.distance), (self.humanGesture.distance /\ self.gestures[index].distance)) distanceDiffRatio = assessments[index][Gesture.totalDistance] /\ min(self.gestures[index].distance, self.humanGesture.distance) if templateGestureRatio < 1.25 and distanceDiffRatio < 2: return index
def classifyGesture(self): minError = 2**31 - 1 # a large value minErrorIndex = -1 self.humanGesture = Gesture(self.gesturePoints, "Human Gesture") likelihoodScores = [0] * len(self.gestures) assessments = [{}] * len(self.gestures) for i in xrange(len(self.gestures)): # print "Calling:", self.gestures[i].name, self.humanGesture.name assessments[i] = Gesture.compareGestures(self.gestures[i], self.humanGesture) # print self.gestures[i].name # print assessments[i] errorList = [ assessments[i][Gesture.totalError] for i in xrange(len(assessments)) ] index = errorList.index(min(errorList)) # Basic elimination to figure out if result is valid templateGestureRatio = max( (self.gestures[index].distance / self.humanGesture.distance), (self.humanGesture.distance / self.gestures[index].distance)) distanceDiffRatio = assessments[index][Gesture.totalDistance] / min( self.gestures[index].distance, self.humanGesture.distance) if templateGestureRatio < 1.25 and distanceDiffRatio < 2: self.gestures[index].action()
def makeCircles(): radius = 512 pointCount = 256 ccwCirclePoints = [(radius*math.cos(t), radius*math.sin(t)) \ for t in np.linspace(0, 2*math.pi, num=pointCount)] ccwCircle = Gesture(ccwCirclePoints, name="CW Circle") cwCirclePoints = [(radius*math.cos(t), -radius*math.sin(t)) \ for t in np.linspace(0, 2*math.pi, num=pointCount)] cwCircle = Gesture(cwCirclePoints, name="CCW Circle") defaultGestures.append(ccwCircle) defaultGestures.append(cwCircle)
def __init__(self): """ Configuration """ # Camera settings self.FRAME_WIDTH = 341 self.FRAME_HEIGHT = 256 self.flip_camera = True # Mirror image self.camera = cv2.VideoCapture(1) # ...you can also use a test video for input #video = "/Users/matthiasendler/Code/snippets/python/tracker/final/assets/test_video/10.mov" #self.camera = cv2.VideoCapture(video) #self.skip_input(400) # Skip to an interesting part of the video if not self.camera.isOpened(): print "couldn't load webcam" return #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, self.FRAME_WIDTH) #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, self.FRAME_HEIGHT) self.filters_dir = "filters/" # Filter settings in trackbar self.filters_file = "filters_default" # Load filter settings current_config = self.filters_dir + self.filters_file self.filters = Filters(current_config) # No actions will be triggered in test mode # (can be used to adjust settings at runtime) self.test_mode = False # Create a hand detector # In fact, this is a wrapper for many detectors # to increase detection confidence self.detector = Detector(self.filters.config) # Knowledge base for all detectors self.kb = KB() # Create gesture recognizer. # A gesture consists of a motion and a hand state. self.gesture = Gesture() # The action module executes keyboard and mouse commands self.action = Action() # Show output of detectors self.output = Output() self.run()
def makeLines(): pointCount = 256 hLineLR = Gesture([(-t, 0) for t in xrange(pointCount)], name="Horizontal Line Right to Left") hLineRL = Gesture([(t, 0) for t in xrange(pointCount)], name="Horizontal Line Left to Right") vLineTB = Gesture([(0, t) for t in xrange(pointCount)], name="Vertical Line Top to Bottom") vLineBT = Gesture([(0, -t) for t in xrange(pointCount)], name="Vertical Line Bottom to Top") diagonalTLtoBR = Gesture([(t, t) for t in xrange(pointCount)], name="Diagonal Top Left to Bottom Right") diagonalBRtoTL = Gesture([(-t, -t) for t in xrange(pointCount)], name="Diagonal Bottom Right to Top Left") diagonalTRtoBL = Gesture([(-t, t) for t in xrange(pointCount)], name="Diagonal Top Right to Bottom Left") diagonalBLtoTR = Gesture([(t, -t) for t in xrange(pointCount)], name="Diagonal Bottom Left to Top Right") defaultGestures.append(hLineLR) defaultGestures.append(hLineRL) defaultGestures.append(vLineTB) defaultGestures.append(vLineBT) defaultGestures.append(diagonalTLtoBR) defaultGestures.append(diagonalBRtoTL) defaultGestures.append(diagonalTRtoBL) defaultGestures.append(diagonalBLtoTR)
def save(self, skeletons): if len(skeletons) < 1: return False for skeleton in skeletons: gesture = Gesture("GESTURE_%d" % (self.__gesture_id_counter), recording=skeleton.gesture_data()) self.__gestures.append(gesture) print("Gesture added: %s" % gesture.id()) self.__gesture_id_counter += 1 if self.matcher: self.matcher.learn(gesture.averages().data(), gesture.id()) return self.__gestures[-1]
def main(): g = Gesture() point_list = [(0,0), (1,0), (1,1), (0,1), (0,2), (2,2), (4,2), (5,3), (4,3), (3,3), (0,3), (1,2), (2,3), (4,5), (2,4), (1,4)] point_list.extend([(point[0]+1, point[1]+2) for point in point_list]) point_list.extend([(point[0]+10, point[1]+20) for point in point_list]) print len(point_list) g.add_stroke(point_list=point_list) g.normalize() points = g.strokes[0].points x = [point.x for point in points] y = [point.y for point in points] plot(x,y, 'bo-') pyp.show()
def loadGesturesFromFile(self): self.gestures = [] self.gestureNames = [] read = "" with open(self.gestureFile, 'r') as fin: read = fin.read() fin.close() data = read.split('\n') # Basic check, should replace later with bytestream instead if len(data) < len(self.gestureHeader): self.loadDefaultGestures() else: gestureName = "" gesturePoints = [] cutoff = len(self.gestureHeader) for item in data: if item[:cutoff] == self.gestureHeader: gestureName = item[cutoff:] elif item == self.gestureEnd: self.gestures.append(Gesture(gesturePoints, gestureName)) self.gestureNames.append(gestureName) gestureName = "" gesturePoints = [] else: gesturePoints.append(map(float, item.split()))
def makeInfinity(): scale = 30 pointCount = 256 lemniscatePoints = [ (((scale * math.sqrt(2) * math.cos(t)) / (math.sin(t)**2 + 1)), -((scale * math.sqrt(2) * math.cos(t) * math.sin(t)) / (math.sin(t)**2 + 1))) for t in np.linspace( math.pi / 2, 2 * math.pi + math.pi / 2, num=pointCount) ] infinity = Gesture(lemniscatePoints, name="Infinity") defaultGestures.append(infinity)
def addRecordedGesture(self): gestureName = "" while True: gestureName = "".join([chr(random.randint(ord('a'), ord('z'))) \ for i in xrange(20)]) if gestureName not in self.getGestureNames(): break newGesture = Gesture(self.gesturePoints, name=gestureName) self.gestures.append(newGesture) print "RECORDED NEW ONE", gestureName self.lastAction = gestureName return gestureName
def __init__(self): """ Configuration """ # Camera settings self.FRAME_WIDTH = 341 self.FRAME_HEIGHT = 256 self.flip_camera = True # Mirror image self.camera = cv2.VideoCapture(1) # ...you can also use a test video for input #video = "/Users/matthiasendler/Code/snippets/python/tracker/final/assets/test_video/10.mov" #self.camera = cv2.VideoCapture(video) #self.skip_input(400) # Skip to an interesting part of the video if not self.camera.isOpened(): print "couldn't load webcam" return #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, self.FRAME_WIDTH) #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, self.FRAME_HEIGHT) self.filters_dir = "filters/" # Filter settings in trackbar self.filters_file = "filters_default" # Load filter settings current_config = self.filters_dir + self.filters_file self.filters = Filters(current_config) # No actions will be triggered in test mode # (can be used to adjust settings at runtime) self.test_mode = False # Create a hand detector # In fact, this is a wrapper for many detectors # to increase detection confidence self.detector = Detector(self.filters.config) # Knowledge base for all detectors self.kb = KB() # Create gesture recognizer. # A gesture consists of a motion and a hand state. self.gesture = Gesture() # The action module executes keyboard and mouse commands self.action = Action() # Show output of detectors self.output = Output() self.run()
def __init__(self, name): self.availableGestures = [ Gesture("rock"), Gesture("paper"), Gesture("scissors"), Gesture("lizard"), Gesture("spock") ] self.chosenGesture = Gesture("NULL") self.name = name self.score = 0
def GetGesture(self): # Retry a few times before initing with palm again timeElapsedSinceLastFoundHand = time.time() - self.timeSinceFoundHandTracked stillTringToFindHandFromTrack = timeElapsedSinceLastFoundHand < config['hand']['timeToKeepSearchingHandWhenLostTracking'] if not stillTringToFindHandFromTrack: self.handTracked = None if self.handTracked is not None or stillTringToFindHandFromTrack: self.FindHandFromTrack() else: self.TryToTrackHand() self.currentGesture = self.handTracked if not self.foundHand: self.currentGesture = Gesture() self.currentGesture.properties['needInitPalm'] = not stillTringToFindHandFromTrack self.SetTimeElapsedSinceSameGesture() return self.currentGesture
def classifyGesture(self): minError = 2**31 - 1 # a large value minErrorIndex = -1 self.humanGesture = Gesture(self.gesturePoints, "Human Gesture") likelihoodScores = [0] * len(self.gestures) assessments = [{}] * len(self.gestures) for i in xrange(len(self.gestures)): assessments[i] = Gesture.compareGestures(self.gestures[i], self.humanGesture) errorList = [assessments[i][Gesture.totalError] \ for i in xrange(len(assessments))] index = errorList.index(min(errorList)) # Basic elimination to figure out if result is valid templateGestureRatio = max((self.gestures[index].distance /\ self.humanGesture.distance), (self.humanGesture.distance /\ self.gestures[index].distance)) distanceDiffRatio = assessments[index][Gesture.totalDistance] /\ min(self.gestures[index].distance, self.humanGesture.distance) if templateGestureRatio < 1.25 and distanceDiffRatio < 2: return index
def predict(filename): gesture = Gesture.from_abs_file(filename)[0] return gesture.predict()
def __init__(self): self.gestures = Gesture() self.gestures.set_values("bestfist.xml", 1.2, 5) self.gestures.set_values("cascade.xml", 1.2, 30) self.gestures.set_values("eye3.xml", 1.1, 36) self.gestures.set_values("face.xml", 1.3, 5)
class CascadeGestureDetector(): def __init__(self): self.gestures = Gesture() self.gestures.set_values("bestfist.xml", 1.2, 5) self.gestures.set_values("cascade.xml", 1.2, 30) self.gestures.set_values("eye3.xml", 1.1, 36) self.gestures.set_values("face.xml", 1.3, 5) def get_last_detected_gesture(self): return self.gestures.gesture_detected_frame def set_perimeters(self, perimeters): self.gestures.set_perimeters(perimeters) def detect_face(self, frame): return (self.gestures.detect_face(frame)) def detect_eyes(self, frame): return (self.gestures.detect_eyes(frame)) def detect_hand_gesture(self, frame): if self.gestures.first_hand_gesture_index_detected == 0: if self.gestures.detect_hand_gesture(frame, 0): return True return self.gestures.detect_hand_gesture(frame, 1) else: if self.gestures.detect_hand_gesture(frame, 1): return True return self.gestures.detect_hand_gesture(frame, 0)
class Tracker(object): """ This is the main program which gives a high-level view of all the running subsystems. It connects camera input with output in form of "actions" (such as keyboard shortcuts on the users behalf). This is done by locating a hand in an image and detecting features, like the number of fingers, and trying to match that data with a known gesture. """ def __init__(self): """ Configuration """ # Camera settings self.FRAME_WIDTH = 341 self.FRAME_HEIGHT = 256 self.flip_camera = True # Mirror image self.camera = cv2.VideoCapture(1) # ...you can also use a test video for input #video = "/Users/matthiasendler/Code/snippets/python/tracker/final/assets/test_video/10.mov" #self.camera = cv2.VideoCapture(video) #self.skip_input(400) # Skip to an interesting part of the video if not self.camera.isOpened(): print "couldn't load webcam" return #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, self.FRAME_WIDTH) #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, self.FRAME_HEIGHT) self.filters_dir = "filters/" # Filter settings in trackbar self.filters_file = "filters_default" # Load filter settings current_config = self.filters_dir + self.filters_file self.filters = Filters(current_config) # No actions will be triggered in test mode # (can be used to adjust settings at runtime) self.test_mode = False # Create a hand detector # In fact, this is a wrapper for many detectors # to increase detection confidence self.detector = Detector(self.filters.config) # Knowledge base for all detectors self.kb = KB() # Create gesture recognizer. # A gesture consists of a motion and a hand state. self.gesture = Gesture() # The action module executes keyboard and mouse commands self.action = Action() # Show output of detectors self.output = Output() self.run() def run(self): """ In each step: Read the input image and keys, process it and react on it (e.g. with an action). """ while True: img = self.get_input() hand = self.process(img) ref = self.action.get_reference_point() self.output.show(img, hand, ref) def process(self, img): """ Process input """ # Run detection hand = self.detector.detect(img) # Store result in knowledge base self.kb.update(hand) if not self.test_mode: # Try to interprete as gesture self.interprete(hand) return hand def interprete(self, hand): """ Try to interprete the input as a gesture """ self.gesture.add_hand(hand) operation = self.gesture.detect_gesture() self.action.execute(operation) def get_input(self): """ Get input from camera and keyboard """ self.get_key() _, img = self.camera.read() img = cv2.resize(img, (self.FRAME_WIDTH, self.FRAME_HEIGHT)) if self.flip_camera: img = cv2.flip(img, 1) return img def get_key(self): """ Read keyboard input """ key = cv2.waitKey(self.filters.config["wait_between_frames"]) if key == ord('+'): # Reduce program speed self.filters.config["wait_between_frames"] += 500 if key == ord('-'): # Increase program speed if self.filters.config["wait_between_frames"] >= 500: self.filters.config["wait_between_frames"] -= 500 #if key == ord('s'): # Save config # self.filters.save() if key == ord('r'): # Reset all detectors self.detector.reset() self.action.reset() if key == ord('d'): # Make a screenshot self.output.make_screenshot() if key == ord('p') or key == ord(' '): # Pause cv2.waitKey() if key == ord('t'): # Test mode self.test_mode = not self.test_mode if key == ord('1'): self.output.toggle_estimate() if key == ord('2'): self.output.toggle_detectors() if key == ord('3'): self.output.toggle_skin() if key == ord('f'): self.toggle_filters() if key == 63235: # Right arrow self.skip_input(20) if key == 27 or key == ord('q'): # Abort program on ESC, q or space exit() def toggle_filters(self): """ Load the next filter settings """ self.filters_file = self.next_filters_file() current_config = self.filters_dir + self.filters_file self.filters.set_config(current_config) def next_filters_file(self): """ Get the next filter settings """ filters = listdir(self.filters_dir) for i, f in enumerate(filters): if f == self.filters_file: return filters[(i + 1) % len(filters)] def skip_input(self, x=1): """ Skip to a different part of a video sequence. """ for i in range(0, x): self.camera.grab()
class Game: def __init__(self): self.define_contestants() self.number_of_rounds = 0 self.gestures = Gesture(self.player_one, self.player_two) def define_contestants(self): # prompts user to pick a human or ai opponent self.opening_statement() opponent = input("\nPress 1 to play against a Human or type anything else to play against an AI" "\n >") if opponent == '1': self.player_one = Human() self.player_two = Human() else: self.player_one = Human() self.player_two = Ai() def run_game(self): self.player_one.select_name() self.player_two.select_name() self.battle() self.display_winner() def battle(self): win_limit = self.best_of() while self.player_one.score < win_limit and self.player_two.score < win_limit: self.player_one.pick_gesture(self.player_one.name) self.player_two.pick_gesture(self.player_two.name) self.display_gestures(self.player_one, self.player_two) while self.player_one.chosen_gesture == self.player_two.chosen_gesture: print("\nDraw! Choose again!") self.number_of_rounds += 1 self.player_one.pick_gesture(self.player_one.name) self.player_two.pick_gesture(self.player_two.name) self.display_gestures(self.player_one, self.player_two) self.decide_round_winner() self.number_of_rounds += 1 def best_of(self): # lets user pick best of however many rounds # chooses best of 3 if user chooses below 3 while True: try: chosen_best_of = input("\nBest of how many rounds? Minimum = 3" "\n>") chosen_best_of = int(chosen_best_of) except ValueError: print("\nMake sure you type a positive integer!") continue except TypeError: print("\nMake sure you type a positive integer!") continue else: if chosen_best_of >= 3 and chosen_best_of % 2 == 0: return (chosen_best_of / 2) + 1 elif chosen_best_of >= 3 and chosen_best_of % 2 == 1: return math.ceil((chosen_best_of / 2)) else: return 2 def ready_check(self): pass def display_winner(self): if self.player_one.score > self.player_two.score: print("\n*************************************") print(f"{self.player_one.name} wins in {self.number_of_rounds} rounds!" f"\nBetter luck next time {self.player_two.name}") print("*************************************") else: print("\n*************************************") print(f"{self.player_two.name} wins in {self.number_of_rounds} rounds!" f"\nBetter luck next time {self.player_one.name}") print("*************************************") def opening_statement(self): # General opening lines to start the game print("\nWelcome to Rock, Paper, Scissors, Lizard, Spock!" "\n\nHere are the rules:" "\n\nRock crushes Scissors" "\nScissors cuts Paper" "\nPaper covers Rock" "\nRock crushes Lizard" "\nLizard poisons Spock" "\nSpock smashes Scissors" "\nScissors decapitates Lizard" "\nLizard eats paper" "\nPaper disproves Spock" "\nSpock vaporizes Rock") def decide_round_winner(self): winner = self.gestures.define_winner(self.player_one.chosen_gesture, self.player_two.chosen_gesture) winner.score += 1 print("--------------------------------------------------") print(f"{winner.name}'s score is: {winner.score}") def display_gestures(self, player1, player2): print(f"\n{player1.name} chooses {player1.chosen_gesture}!") print(f"{player2.name} chooses {player2.chosen_gesture}!") print("--------------------------------------------------")
#!/usr/bin/env python3 import os, sys, time import numpy as np from gesture import Gesture pos_files = [x for x in os.listdir(os.getcwd()) if ".pos" in x] print(pos_files) gestures = [] for pos in pos_files: gestures.append(Gesture(pos)) for gesture in gestures: print(gesture.name, gesture.filename) lock = time.time() for line in sys.stdin: line = line.rstrip().split(',') if line[0] == 'ORT': for gesture in gestures: if gesture.in_position(float(line[1]), float(line[2]), float(line[3])): if time.time() - 1.5 > gesture.lock: if gesture.name == 'time': os.system("say it is now `date +%I`, `date +%M`") print('time gesture') elif gesture.name == 'dab':
class Server(Thread): """Thread-based server for midi input, output, and gesture processing""" def __init__(self, in_device, in_port, out_device, out_port): Thread.__init__(self) self.setDaemon(True) # ports self.in_port = in_port self.in_device = in_device self.in_name = in_device.getPortName(in_port) self.out_port = out_port self.out_device = out_device self.out_name = out_device.getPortName(out_port) # gesture self.gesture = None self.g_start = None self.g_last = None self.g_ongoing = False self.g_break = 1.0 # state self.sustained = [False] * 120 self.pedal = False self.quit = False def run(self): """The server's main loop""" Xs = [] self.in_device.openPort(self.in_port) self.out_device.openPort(self.out_port) self.in_device.ignoreTypes(True, False, True) while True: if self.quit: return msg = self.in_device.getMessage() if msg: self.add_to_gesture(msg) if self.g_ongoing and not any( self.sustained ) and not self.pedal and time.time() - self.g_last > self.g_break: self.g_ongoing = False c = Collecture(self.gesture) c = randomize(c) c.play(self.out_device) # compress and decompress gesture # g = Gesture(self.gesture.dist_vector(), "dist") # play decompressed vector # g.play(self.out_device) def add_to_gesture(self, msg): """Add messages to ongoing gesture""" # start a new gesture if one is not ongoing if not self.g_ongoing: self.gesture = Gesture() self.g_start = time.time() self.g_ongoing = True # set sustain pedal control if msg.isController() and msg.getControllerNumber() == 64: if msg.getControllerValue() == 127: self.pedal = True else: self.pedal = False # mark note as sustained or completed if msg.isNoteOn(): self.sustained[msg.getNoteNumber()] = True if msg.isNoteOff(): self.sustained[msg.getNoteNumber()] = False # add midi message and time since start to gesture self.gesture.add_message(msg, time.time() - self.g_start) # record event time as last to inform gesture ending self.g_last = time.time() print_message(msg, self.in_name) def end(self): self.quit = True
class Tracker(object): """ This is the main program which gives a high-level view of all the running subsystems. It connects camera input with output in form of "actions" (such as keyboard shortcuts on the users behalf). This is done by locating a hand in an image and detecting features, like the number of fingers, and trying to match that data with a known gesture. """ def __init__(self): """ Configuration """ # Camera settings self.FRAME_WIDTH = 341 self.FRAME_HEIGHT = 256 self.flip_camera = True # Mirror image self.camera = cv2.VideoCapture(1) # ...you can also use a test video for input #video = "/Users/matthiasendler/Code/snippets/python/tracker/final/assets/test_video/10.mov" #self.camera = cv2.VideoCapture(video) #self.skip_input(400) # Skip to an interesting part of the video if not self.camera.isOpened(): print "couldn't load webcam" return #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, self.FRAME_WIDTH) #self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, self.FRAME_HEIGHT) self.filters_dir = "filters/" # Filter settings in trackbar self.filters_file = "filters_default" # Load filter settings current_config = self.filters_dir + self.filters_file self.filters = Filters(current_config) # No actions will be triggered in test mode # (can be used to adjust settings at runtime) self.test_mode = False # Create a hand detector # In fact, this is a wrapper for many detectors # to increase detection confidence self.detector = Detector(self.filters.config) # Knowledge base for all detectors self.kb = KB() # Create gesture recognizer. # A gesture consists of a motion and a hand state. self.gesture = Gesture() # The action module executes keyboard and mouse commands self.action = Action() # Show output of detectors self.output = Output() self.run() def run(self): """ In each step: Read the input image and keys, process it and react on it (e.g. with an action). """ while True: img = self.get_input() hand = self.process(img) ref = self.action.get_reference_point() self.output.show(img, hand, ref) def process(self, img): """ Process input """ # Run detection hand = self.detector.detect(img) # Store result in knowledge base self.kb.update(hand) if not self.test_mode: # Try to interprete as gesture self.interprete(hand) return hand def interprete(self, hand): """ Try to interprete the input as a gesture """ self.gesture.add_hand(hand) operation = self.gesture.detect_gesture() self.action.execute(operation) def get_input(self): """ Get input from camera and keyboard """ self.get_key() _, img = self.camera.read() img = cv2.resize(img, (self.FRAME_WIDTH, self.FRAME_HEIGHT)) if self.flip_camera: img = cv2.flip(img, 1) return img def get_key(self): """ Read keyboard input """ key = cv2.waitKey(self.filters.config["wait_between_frames"]) if key == ord('+'): # Reduce program speed self.filters.config["wait_between_frames"] += 500 if key == ord('-'): # Increase program speed if self.filters.config["wait_between_frames"] >= 500: self.filters.config["wait_between_frames"] -= 500 #if key == ord('s'): # Save config # self.filters.save() if key == ord('r'): # Reset all detectors self.detector.reset() self.action.reset() if key == ord('d'): # Make a screenshot self.output.make_screenshot() if key == ord('p') or key == ord(' '): # Pause cv2.waitKey() if key == ord('t'): # Test mode self.test_mode = not self.test_mode if key == ord('1'): self.output.toggle_estimate() if key == ord('2'): self.output.toggle_detectors() if key == ord('3'): self.output.toggle_skin() if key == ord('f'): self.toggle_filters() if key == 63235: # Right arrow self.skip_input(20) if key == 27 or key == ord('q'): # Abort program on ESC, q or space exit() def toggle_filters(self): """ Load the next filter settings """ self.filters_file = self.next_filters_file() current_config = self.filters_dir + self.filters_file self.filters.set_config(current_config) def next_filters_file(self): """ Get the next filter settings """ filters = listdir(self.filters_dir) for i, f in enumerate(filters): if f == self.filters_file: return filters[(i+1) % len(filters)] def skip_input(self, x=1): """ Skip to a different part of a video sequence. """ for i in range(0,x): self.camera.grab()
import os from datalist import DataList from names import shortNames from gesture import Gesture from matplotlib import pyplot as plt name = "dtap" gest = Gesture("data/40_" + name + ".gest") gest.getFeatureSet() plt.show() # dl = DataList() # name = "waveout" # for i in range(7, 17): # gest = Gesture("data/"+str(i)+"_"+name+".gest") # gest.plotSpecgram() # print str(i) + ":" + str(gest.getFeatureSet()[80:]) # plt.show() # for i in range(17, 27): # gest = Gesture("data/"+str(i)+"_"+name+".gest") # gest.plotSpecgram() # print str(i) + ":" + str(gest.getFeatureSet()[80:]) # plt.show() # for i in range(27, 37): # gest = Gesture("data/"+str(i)+"_"+name+".gest")
class Game: def __init__(self): self.gesture = Gesture() self.run() def run(self): done = False self.reset() while not done: self.takeTurn() if self.gameOver(): value = input("Would you like to play again? y/n\n") while value != "y" and value != "n": os.system('cls') value = input("Would you like to play again? y/n\n") if value == 'n': done = True else: self.reset() else: print( f"Player1: {self.p1}, Player2: {self.p2}. {self.rtw} points to win" ) input("Press enter to continue\n") def gameOver(self): if self.p1 == self.rtw: print("Player 1 is the winner!\n") return True elif self.p2 == self.rtw: print("Player 2 is the winner!\n") return True return False def reset(self): os.system('cls') self.welcome() self.vsNPC = self.selectMode() self.rtw = self.roundsToWin() self.p1 = 0 self.p2 = 0 def takeTurn(self): os.system('cls') p1_action = self.player1.takeAction() os.system('cls') p2_action = self.player2.takeAction() os.system('cls') print(f"player1 chose {p1_action}, player2 chose {p2_action}.") result = self.gesture.compare_value(p1_action, p2_action) print(result[1]) if result[0] == -1: self.p1 += 1 print("Player 1 wins this round") elif result[0] == 1: self.p2 += 1 print("Player 2 wins this round") def roundsToWin(self): os.system('cls') value = input( "Please enter a number 2-9 for number of rounds to win\n") valid = set(list("23456789")) while (value not in valid): os.system('cls') value = input( "Invalid input: Please enter a number 2-9 for number of rounds to win\n" ) return int(value) def selectMode(self): value = input("Please enter 1 for vs npc, or 2 for player vs player\n") while value != "1" and value != "2": os.system('cls') value = input( "Invalid input. Please enter 1 for vs npc, or 2 for player vs player\n" ) self.vsNPC = value == "1" self.player1 = HumanPlayer(1) if self.vsNPC: self.player2 = NPC(2) else: self.player2 = HumanPlayer(2) def welcome(self): print("Welcome to Rock Paper Scissors Lizard Spock!") print("Rock crushes Scissors, Scissors cuts Paper, Paper covers Rock") print( "Rock rushes Lizard, Lizard poisons Spock, Spock smashes Scissors") print( "Scissors decapitates Lizard, Lizard eats paper, Paper disproves Spock," ) print("and Spock vaporizes rock") print()
# read from file with open('../data/Data/0' + str(i) + '/glasses/raw_2_0_unsync.txt') as f: x = f.readlines() data = [] for e in x: data.append(map(float, e[:-1].split(' ')[1:-1])) #print len(data), len(data[0]), data[0] # --- get a gesture chunk --- d_obj = Descent() g_obj = Gesture() swipe_threshold = int(d_obj.median(data[0])) * 0.80 touch_threshold = int(d_obj.median(data[0])) * 0.10 #swipe_threshold, touch_threshold = d_obj.auto_thresh(data) y = g_obj.gd(data, swipe_threshold, 2) if len(y) > 3: print '~~~~\nSwipe Detected, Swipe gesture direction\n' print len(y), y else: y = g_obj.gd_double_touch(data, touch_threshold, 0) if y: print '~~~~\nTouch Detected, index of touch\n' print len(y), y
from gesture import Gesture # g = Gesture.from_file("круг.json") # # print('g', g) # print('gm', g.measurements) # print('proj', g.select_proj_on_plane()) # print('res', g.select_proj_2d()) import matplotlib.pyplot as plt g = Gesture.from_file("cirl_test.json")[0] # g.draw_accel_3d() # g.draw_accel_3d(with_surf=True) # g.to_image() g.predict() # data = g.select_proj_2d() # plt.scatter(data[:,0], data[:,1]) # plt.show()
def __init__(self): self.process_manager = ProcessManager() self.fist = Gesture("fist.xml") self.palm = Gesture("palm.xml")
import numpy as np import os from names import * from gesture import Gesture print("gesture|avg |had |u |d |l |r ") print("-------+----+----+--+--+--+--") for name in shortNames: total = 0 had = 0 angles = [0, 0, 0, 0] #up, down, left, right for i in range(7, 37): #Get gestures gest = Gesture("data/" + str(i) + "_" + name + ".gest", True) #Get the total num of dir samples total += gest.numDirSamples #if there were inc had counter if gest.numDirSamples > 0: had += 1 #Get general directions for x in range(gest.numDirSamples): angle = gest.sgestures[x][2] if angle >= 45 and angle < 135: angles[3] += 1 elif angle >= 135 and angle < 225: angles[0] += 1 elif angle >= 225 and angle < 315: angles[2] += 1
def analyse(bw, image): """ Main logic. """ bw = cv2.pyrUp(bw) contours, hierarchy = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) biggest = find_biggest_contour(contours) if biggest is not -1: gesture = Gesture() gesture.contours = contours gesture.biggest = biggest gesture.bounding = bounding_rect(contours[biggest]) gesture.hull_p = cv2.convexHull(contours[biggest]) gesture.hull_i = cv2.convexHull(contours[biggest], returnPoints=False) gesture.hull_p = cv2.approxPolyDP(gesture.hull_p, 18, True) if len(contours[biggest]) > 3: gesture.defects = cv2.convexityDefects(contours[biggest], gesture.hull_i) gesture.check_convexity() is_hand = gesture.is_hand() if is_hand: global counter, clicked dx = abs(gesture.bounding[0] - gesture.bounding[2]) dy = abs(gesture.bounding[1] - gesture.bounding[3]) #print abs(dx - dy) #print dy * 0.3 tresh = dy * 0.3 if abs(dx - dy) < tresh and counter == 0: click() print "click" counter += 1 if abs(dx - dy) < tresh: counter += 1 if counter == 20: counter = 0 if abs(dx - dy) > tresh: sh = bw.shape width = sh[0] height = sh[1] move_mouse(gesture.get_center(), height, width) draw_contours(image, gesture) return image
def load(self): self.gestures = {} for name in shortNames: self.gestures[name] = Gesture( "data/" + str(self.id) + "_" + name + ".gest", True)
def __init__(self): self.define_contestants() self.number_of_rounds = 0 self.gestures = Gesture(self.player_one, self.player_two)
def __init__(self): self.gesture = Gesture() self.run()