Пример #1
0
    def _do_participant_practice_turn(self):
        self.websocket_server.send("hide_answers()")

        # Some introduction by the robot
        self.robot_connection.say(Speech.get(self.lang, 'practice_participant_instruction'))
        self.robot_connection.say(Speech.get(self.lang, 'practice_participant_instruction_2'))

        # The concept to perform is always 'ball' in the practice round
        # When adding a new language, don't forget to translate this term.
        concept = {
            'id': 'ball',
            'description': {
                'nl': 'Bal',
                'en': 'Ball'
            },
            'article': {
                'nl': 'Een',
                'en': 'A'
            },
            'category': 'entertainment',
            'image_filename': 'ball.png'
        }

        self._log("participant_practice_turn()")

        # Show the ball on the tablet screen so the participant knows
        # what to perform a gesture for.
        self.websocket_server.send("show_item(" + json.dumps(concept) + ")")
Пример #2
0
    def _do_robot_second_turn(self):
        self._log("robot_second_turn()")
        self.robot_connection.say(Speech.get(self.lang, 'robot_turn_start'))

        self._pick_and_perform()

        self.websocket_server.send("enable_answers()")
Пример #3
0
    def _do_robot_turn(self):
        # Hide previous results screen from the participant's turn (if applicable)
        self.websocket_server.send("hide_classification()")
        self.num_answer_attempts = 0
        self.current_concept = self.concepts_robot[self.concept_index]
        self.concepts_used.append(self.current_concept)

        print "Showing: " + self.current_concept["id"]

        if self.concept_index < (self.num_concepts-1) and self.concept_index > 0:
            self.robot_connection.say(Speech.get(self.lang, 'robot_turn'))

        # The robot indicates that we're about to start the last round
        elif self.concept_index == (self.num_concepts - 1):
            self.robot_connection.say(Speech.get(self.lang, 'robot_last_turn'))

        self.robot_connection.say(Speech.get(self.lang, 'robot_turn_start'))

        self._pick_and_perform()

        # Randomly pick three distractor objects. At the moment they can be any of the available concepts.
        answers = list()
        answers.append(self.current_concept)

        for i in range(0, 3):
            answer = Concepts.items[random.randint(0, len(Concepts.items)-1)]

            while answer in answers or answer in self.concepts_used:
                answer = Concepts.items[random.randint(0, len(Concepts.items)-1)]

            answers.append(answer)

        # Shuffle answers
        random.shuffle(answers)

        answer_data = dict()
        answer_data["answers"] = answers
        answer_data["correct_answer"] = self.current_concept["id"]

        # Send them to the tablet
        self.websocket_server.send("show_answers(" + json.dumps(answer_data) + ")")
        self._log("robot_turn(concept=" + self.current_concept["id"] + ",answers_presented=" + json.dumps(answer_data) + ")")

        self.robot_connection.say(Speech.get(self.lang, 'robot_turn_end'))
Пример #4
0
    def _introduction(self):
        self.concept_index = 0
        self._pick_concepts()
        self._log("concepts_robot(" + json.dumps(self.concepts_robot) + ")")
        self._log("concepts_participant(" + json.dumps(self.concepts_participant) + ")")

        # The record-only mode is implemented at some point to quickly make
        # a set of initial recordings. Has not been used since ;)
        if self.record_only:
            self._do_participant_turn()
        else:
            # The robot will introduce itself and the game
            self.robot_connection.say(Speech.get(self.lang, 'intro'))
            self.is_practice = True
            # Then the robot will start with a practice run, where it will
            # perform the gesture for the concept 'glasses'
            self._do_robot_practice_turn()
Пример #5
0
    def _do_participant_turn(self):
        self.previous_guess = None
        self.previous_recording = None

        # Hide the previous screen on the tablet (if needed)
        self.websocket_server.send("hide_answers()")
        self.num_answer_attempts = 0

        # Pick a concept that the participant should perform
        self.current_concept = self.concepts_participant[self.concept_index]
        self.concepts_used.append(self.current_concept)

        if not self.record_only:
            self.robot_connection.say(Speech.get(self.lang, 'participant_turn'))

        # Send concept to the web client
        self.websocket_server.send("show_item(" + json.dumps(self.current_concept) + ")")
        self._log("participant_turn(" + self.current_concept['id'] + ")")
Пример #6
0
    def recording_completed(self, concept, filename):
        print "Recording completed: " + filename
        self._log('recording_completed(concept=' + concept + ',filename=' + filename + ')')

        # Temporary path introduced to efficiently record gestures without
        # completing the entire game.
        if self.record_only:
            self.concept_index += 1

            if self.concept_index < len(self.concepts_participant):
                self._do_participant_turn()

        # "Normal" situation, but in a practice round
        elif self.is_practice:
            # The robot will always guess correctly in the practice round
            self.robot_connection.happy_feedback()
            self.robot_connection.say(Speech.get(self.lang, 'practice_participant_feedback'))

            # Also introduce the feedback on the classification that is shown on the tablet
            guesses = list()
            guesses.append({
                'item': {
                    'id': 'ball',
                    'description': {
                        'nl': 'Bal',
                        'en': 'Ball'
                    },
                    'article': {
                        'nl': 'Een',
                        'en': 'A'
                    },
                    'category': 'entertainment',
                    'image_filename': 'ball.png'
                },
                'percentage': 0.85
            })
            guesses.append({
                'item': {
                    'id': 'windmill',
                    'description': {
                        'nl': 'Molen',
                        'en': 'Mill'
                    },
                    'article': {
                        'nl': 'Een',
                        'en': 'A'
                    },
                    'category': 'static',
                    'image_filename': 'windmill.png'
                },
                'percentage': 0.15
            })
            self.robot_connection.say(Speech.get(self.lang, "practice_participant_explanation_classification_1"))

            # This shows the top 3 of candidates for the robot's guess
            self.websocket_server.send("show_classification(" + json.dumps(guesses) + ")")
            self.robot_connection.say(Speech.get(self.lang, "practice_participant_explanation_classification_2"))

            time.sleep(2)

            self.is_practice = False
            self.robot_connection.say(Speech.get(self.lang, "practice_participant_finished"))

            # Now start the actual game, the robot will go first.
            self._do_robot_turn()

        else:
            # First guess and then add it to the dataset, that's the fair way to go :-)

            # Load the newly recorded gesture
            importer = OurImport()
            loaded_data = importer.load(self.data_dir + "/recordings/" + concept + "/" + filename + ".csv")

            # Extract features from it (inflection points and peaks, mapped to relative locations of body joints)
            fe = GestureFeatureExtractor()
            gist = fe.get_gesture_features_as_string(loaded_data)
            gist_list = gist.split(';')

            # Perform the classification (k-NN)
            guess_obj = self.classifier.classify(gist_list, self.previous_guess)
            neighbors = guess_obj[1]
            guess_obj = guess_obj[0]

            print neighbors

            self._log('robot_guess(correct_answer=' + self.current_concept['id'] + ',given=' + guess_obj[0][0] + ',guesses=' + json.dumps(guess_obj) + ',neighbors=' + json.dumps(neighbors) + ')')

            # Retrieve proper descriptions of objects -- these are the top 5 (at most) guesses that will be displayed
            guesses = list()
            for go in guess_obj:
                guesses.append({
                    'item': Concepts.find(go[0]),
                    'percentage': go[1]
                })

            guess = Speech.get(self.lang, 'robot_guess')
            obj = guesses[0]['item']
            guess = guess.replace('[obj]', obj["article"][self.lang] + " " + obj["description"][self.lang])

            # Show the top 5 (at most) candidates, then have the robot announce its #1 guess
            self.websocket_server.send("show_classification(" + json.dumps(guesses) + ")")
            self.robot_connection.say(guess)

            # Pause for dramatic effect...
            time.sleep(2)

            # The robot guessed correctly!
            if guess_obj[0][0] == self.current_concept['id']:
                fb = Speech.get(self.lang, 'positive_feedback_self')
                fb = fb.replace('[obj]', self.current_concept["article"][self.lang] + " " + self.current_concept['description'][self.lang])
                self.num_correct_robot += 1

                self.total_rounds_robot += 1 # it is a robot guessing here
                self.total_wins_robot += 1
                self._save_totals()

                # Announce the robot's victory
                self.robot_connection.happy_feedback()
                self.robot_connection.say(fb)

                # Now we can add it to the existing dataset (for future classification and generation)
                if self.num_answer_attempts == 1:
                    self.gist_dataset.append(self.previous_recording)

                self.gist_dataset.append(gist)
                self.gist_dataset.save("data/gists.csv")
                self._regenerate_cluster(self.current_concept['id'])

                # Move on
                self.concept_index += 1
                self.websocket_server.send("hide_item()")

                print str(self.concept_index) + " " + str(len(self.concepts_participant)) + " " + str(len(self.concepts_robot))

                # Check whether we've covered all 5 items, then end the session
                if self.concept_index == len(self.concepts_participant):
                    self._log('experiment_end(robot_score=' + str(self.num_correct_robot) + ',participant_score=' + str(self.num_correct_participant) + ')')

                    # Announce the end of the session
                    self.robot_connection.say(Speech.get(self.lang, 'outro_1'))

                    # Depending on who guessed most gestures correctly (this is very likely the participant),
                    # we have a slightly different outro
                    if self.num_correct_robot > self.num_correct_participant:
                        self.robot_connection.say(Speech.get(self.lang, 'outro_robot_wins'))
                    else:
                        self.robot_connection.say(Speech.get(self.lang, 'outro_participant_wins'))

                    self.robot_connection.say(Speech.get(self.lang, 'outro_2'))

                    # Clean up
                    self.websocket_server.send("hide_classification()")
                    self.socket_server.send("controlpanel", "ExperimentFinished()")

                    # Robot can rest
                    self.robot_connection.end_experiment()

                else: # This was not the last round, so we proceed with the next turn for the robot to perform
                    self._do_robot_turn()

            # The robot guessed incorrectly!
            else:
                self.total_rounds_robot += 1 # it is a robot guessing
                self._save_totals()

                self.robot_connection.say(Speech.get(self.lang, 'negative_feedback_self'))
                self.num_answer_attempts += 1

                # If this is the first attempt, try again
                if self.num_answer_attempts == 1:
                    self.robot_connection.say(Speech.get(self.lang, 'participant_try_again'))
                    self.websocket_server.send("hide_classification()")
                    self.previous_guess = guess_obj[0][0]
                    self.previous_recording = gist
                    # The participant will be asked to perform another gesture for the same item
                    self._do_participant_second_turn()

                # Otherwise, call it quits and move on :-)
                else:
                    self.concept_index += 1
                    self.websocket_server.send("hide_item()")

                    # Now we can add it to the existing dataset (for future classification and generation)
                    self.gist_dataset.append(self.previous_recording)
                    self.gist_dataset.append(gist)
                    self.gist_dataset.save("data/gists.csv")
                    self._regenerate_cluster(self.current_concept['id'])

                    print str(self.concept_index) + " " + str(len(self.concepts_participant)) + " " + str(len(self.concepts_robot))

                    # Check whether we've covered all 5 items, then end the session
                    if self.concept_index == len(self.concepts_participant):
                        self.robot_connection.say(Speech.get(self.lang, 'outro_1'))

                        # Depending on who guessed most gestures correctly (this is very likely the participant),
                        # we have a slightly different outro
                        if self.num_correct_robot > self.num_correct_participant:
                            self.robot_connection.say(Speech.get(self.lang, 'outro_robot_wins'))
                        else:
                            self.robot_connection.say(Speech.get(self.lang, 'outro_participant_wins'))

                        self.robot_connection.say(Speech.get(self.lang, 'outro_2'))

                        # Clean up
                        self.websocket_server.send("hide_classification()")

                        # Robot can rest
                        self.robot_connection.end_experiment()

                    else: # This was not the last round, so we proceed with the next turn for the robot to perform
                        self._do_robot_turn()
Пример #7
0
    def answer_given(self, answer):
        print "Answer received: " + answer

        # If this is a practice run, the correct answer is always 'glasses'
        if self.is_practice:
            self._log("practice_answer_given(correct=glasses,given=" + answer + ")")
            if answer == "glasses": # Correct answer
                self.robot_connection.happy_feedback()
                self.robot_connection.say(Speech.get(self.lang, 'practice_robot_positive_feedback'))
            else: # Incorrect answer
                self.robot_connection.say(Speech.get(self.lang, 'practice_robot_negative_feedback'))

            # In the practice run, regardless of whether the answer was correct,
            # we always move on to the next step (second part of practice run)
            self._do_participant_practice_turn()

        # Correct answer -- increase weights for the performed recording
        elif answer == self.current_concept['id']:
            self._log("answer_given(correct=" + self.current_concept['id'] + ",given=" + answer + ")")
            if self.num_answer_attempts == 0: # Got it right the first time
                self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] += 1
                self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] += 1
            else: # Got it right on the second attempt -- less of an increase to the weights because higher chance of guessing correctly
                self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] += 0.75
                self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] += 0.75

            fb = Speech.get(self.lang, 'positive_feedback')
            fb = fb.replace('[obj]', self.current_concept['article'][self.lang] + " " + self.current_concept['description'][self.lang])
            self.num_correct_participant += 1

            self.total_rounds_participant += 1 # it is a participant guessing
            self.total_wins_participant += 1
            self._save_totals() # Keep track of the scores

            self.robot_connection.happy_feedback()
            self.robot_connection.say(fb)
            self._do_participant_turn() # Robot turn over, on to the participant's turn

        # Incorrect answer -- decrease weights for the performed recording
        else:
            self._log("answer_given(correct=" + self.current_concept['id'] + ",given=" + answer + ")")
            self.num_answer_attempts += 1

            self.total_rounds_participant += 1 # it is a participant guessing
            self._save_totals()

            neg_feedback = Speech.get(self.lang, 'negative_feedback')
            neg_feedback = neg_feedback.replace('[obj]', Concepts.find(answer)["description"][self.lang])
            self.robot_connection.say(neg_feedback)

            if self.num_answer_attempts == 1:
                self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] -= 1
                self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] -= 1
                self.robot_connection.say(Speech.get(self.lang, 'robot_try_again'))
                self._do_robot_second_turn()
            else:
                self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] -= 0.75
                self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] -= 0.75
                correct_answer = Speech.get(self.lang, 'correct_answer')
                correct_answer = correct_answer.replace('[obj]', self.current_concept["article"][self.lang] + " " + self.current_concept["description"][self.lang])
                self.robot_connection.say(correct_answer)
                self._do_participant_turn()
Пример #8
0
    def _do_robot_practice_turn(self):
        self.robot_connection.say(Speech.get(self.lang, 'practice_robot_intro'))

        # This loads a recorded gesture (.csv format, from Kinect)
        # and plays it back. In the practice round, we always use the same one.
        importer = OurImport()
        gesture = importer.load(self.data_dir + "/practice/glasses/glasses_practice.csv")
        self.robot_connection.perform_gesture(gesture)

        # These are the answers included only for the practice round
        # If you want to add another language, don't forget to translate the
        # terms here. These are not part of the recorded set of 35 items.
        answer_data = dict()
        answer_data["correct_answer"] = "glasses"

        answer_data["answers"] = [
            {
                'id': 'windmill',
                'description': {
                    'nl': 'Molen',
                    'en': 'Mill'
                },
                'article': {
                    'nl': 'Een',
                    'en': 'A'
                },
                'category': 'static',
                'image_filename': 'windmill.png'
            },
            {
                'id': 'glasses',
                'description': {
                    'nl': 'Bril',
                    'en': 'Glasses'
                },
                'article': {
                    'nl': 'Een',
                    'en': 'A'
                },
                'category': 'tools',
                'image_filename': 'glasses.png'
            },
            {
                'id': 'elephant',
                'description': {
                    'nl': 'Olifant',
                    'en': 'Elephant'
                },
                'article': {
                    'nl': 'Een',
                    'en': 'An'
                },
                'category': 'animate',
                'image_filename': 'elephant.png'
            },
            {
                'id': 'teapot',
                'description': {
                    'nl': 'Theepot',
                    'en': 'Teapot'
                },
                'article': {
                    'nl': 'Een',
                    'en': 'A'
                },
                'category': 'tools',
                'image_filename': 'teapot.png'
            }
        ]

        # Present answers to the participant on the tablet
        self.websocket_server.send("show_answers(" + json.dumps(answer_data) + ")")
        self._log("robot_practice_turn()")
        self.robot_connection.say(Speech.get(self.lang, 'practice_robot_instruction'))