def _do_participant_practice_turn(self):
self.websocket_server.send("hide_answers()")
# Some introduction by the robot
self.robot_connection.say(Speech.get(self.lang, 'practice_participant_instruction'))
self.robot_connection.say(Speech.get(self.lang, 'practice_participant_instruction_2'))
# The concept to perform is always 'ball' in the practice round
# When adding a new language, don't forget to translate this term.
concept = {
'id': 'ball',
'description': {
'nl': 'Bal',
'en': 'Ball'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'entertainment',
'image_filename': 'ball.png'
}
self._log("participant_practice_turn()")
# Show the ball on the tablet screen so the participant knows
# what to perform a gesture for.
self.websocket_server.send("show_item(" + json.dumps(concept) + ")")
def _do_robot_second_turn(self):
self._log("robot_second_turn()")
self.robot_connection.say(Speech.get(self.lang, 'robot_turn_start'))
self._pick_and_perform()
self.websocket_server.send("enable_answers()")
def _do_robot_turn(self):
# Hide previous results screen from the participant's turn (if applicable)
self.websocket_server.send("hide_classification()")
self.num_answer_attempts = 0
self.current_concept = self.concepts_robot[self.concept_index]
self.concepts_used.append(self.current_concept)
print "Showing: " + self.current_concept["id"]
if self.concept_index < (self.num_concepts-1) and self.concept_index > 0:
self.robot_connection.say(Speech.get(self.lang, 'robot_turn'))
# The robot indicates that we're about to start the last round
elif self.concept_index == (self.num_concepts - 1):
self.robot_connection.say(Speech.get(self.lang, 'robot_last_turn'))
self.robot_connection.say(Speech.get(self.lang, 'robot_turn_start'))
self._pick_and_perform()
# Randomly pick three distractor objects. At the moment they can be any of the available concepts.
answers = list()
answers.append(self.current_concept)
for i in range(0, 3):
answer = Concepts.items[random.randint(0, len(Concepts.items)-1)]
while answer in answers or answer in self.concepts_used:
answer = Concepts.items[random.randint(0, len(Concepts.items)-1)]
answers.append(answer)
# Shuffle answers
random.shuffle(answers)
answer_data = dict()
answer_data["answers"] = answers
answer_data["correct_answer"] = self.current_concept["id"]
# Send them to the tablet
self.websocket_server.send("show_answers(" + json.dumps(answer_data) + ")")
self._log("robot_turn(concept=" + self.current_concept["id"] + ",answers_presented=" + json.dumps(answer_data) + ")")
self.robot_connection.say(Speech.get(self.lang, 'robot_turn_end'))
def _introduction(self):
self.concept_index = 0
self._pick_concepts()
self._log("concepts_robot(" + json.dumps(self.concepts_robot) + ")")
self._log("concepts_participant(" + json.dumps(self.concepts_participant) + ")")
# The record-only mode is implemented at some point to quickly make
# a set of initial recordings. Has not been used since ;)
if self.record_only:
self._do_participant_turn()
else:
# The robot will introduce itself and the game
self.robot_connection.say(Speech.get(self.lang, 'intro'))
self.is_practice = True
# Then the robot will start with a practice run, where it will
# perform the gesture for the concept 'glasses'
self._do_robot_practice_turn()
def _do_participant_turn(self):
self.previous_guess = None
self.previous_recording = None
# Hide the previous screen on the tablet (if needed)
self.websocket_server.send("hide_answers()")
self.num_answer_attempts = 0
# Pick a concept that the participant should perform
self.current_concept = self.concepts_participant[self.concept_index]
self.concepts_used.append(self.current_concept)
if not self.record_only:
self.robot_connection.say(Speech.get(self.lang, 'participant_turn'))
# Send concept to the web client
self.websocket_server.send("show_item(" + json.dumps(self.current_concept) + ")")
self._log("participant_turn(" + self.current_concept['id'] + ")")
def recording_completed(self, concept, filename):
print "Recording completed: " + filename
self._log('recording_completed(concept=' + concept + ',filename=' + filename + ')')
# Temporary path introduced to efficiently record gestures without
# completing the entire game.
if self.record_only:
self.concept_index += 1
if self.concept_index < len(self.concepts_participant):
self._do_participant_turn()
# "Normal" situation, but in a practice round
elif self.is_practice:
# The robot will always guess correctly in the practice round
self.robot_connection.happy_feedback()
self.robot_connection.say(Speech.get(self.lang, 'practice_participant_feedback'))
# Also introduce the feedback on the classification that is shown on the tablet
guesses = list()
guesses.append({
'item': {
'id': 'ball',
'description': {
'nl': 'Bal',
'en': 'Ball'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'entertainment',
'image_filename': 'ball.png'
},
'percentage': 0.85
})
guesses.append({
'item': {
'id': 'windmill',
'description': {
'nl': 'Molen',
'en': 'Mill'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'static',
'image_filename': 'windmill.png'
},
'percentage': 0.15
})
self.robot_connection.say(Speech.get(self.lang, "practice_participant_explanation_classification_1"))
# This shows the top 3 of candidates for the robot's guess
self.websocket_server.send("show_classification(" + json.dumps(guesses) + ")")
self.robot_connection.say(Speech.get(self.lang, "practice_participant_explanation_classification_2"))
time.sleep(2)
self.is_practice = False
self.robot_connection.say(Speech.get(self.lang, "practice_participant_finished"))
# Now start the actual game, the robot will go first.
self._do_robot_turn()
else:
# First guess and then add it to the dataset, that's the fair way to go :-)
# Load the newly recorded gesture
importer = OurImport()
loaded_data = importer.load(self.data_dir + "/recordings/" + concept + "/" + filename + ".csv")
# Extract features from it (inflection points and peaks, mapped to relative locations of body joints)
fe = GestureFeatureExtractor()
gist = fe.get_gesture_features_as_string(loaded_data)
gist_list = gist.split(';')
# Perform the classification (k-NN)
guess_obj = self.classifier.classify(gist_list, self.previous_guess)
neighbors = guess_obj[1]
guess_obj = guess_obj[0]
print neighbors
self._log('robot_guess(correct_answer=' + self.current_concept['id'] + ',given=' + guess_obj[0][0] + ',guesses=' + json.dumps(guess_obj) + ',neighbors=' + json.dumps(neighbors) + ')')
# Retrieve proper descriptions of objects -- these are the top 5 (at most) guesses that will be displayed
guesses = list()
for go in guess_obj:
guesses.append({
'item': Concepts.find(go[0]),
'percentage': go[1]
})
guess = Speech.get(self.lang, 'robot_guess')
obj = guesses[0]['item']
guess = guess.replace('[obj]', obj["article"][self.lang] + " " + obj["description"][self.lang])
# Show the top 5 (at most) candidates, then have the robot announce its #1 guess
self.websocket_server.send("show_classification(" + json.dumps(guesses) + ")")
self.robot_connection.say(guess)
# Pause for dramatic effect...
time.sleep(2)
# The robot guessed correctly!
if guess_obj[0][0] == self.current_concept['id']:
fb = Speech.get(self.lang, 'positive_feedback_self')
fb = fb.replace('[obj]', self.current_concept["article"][self.lang] + " " + self.current_concept['description'][self.lang])
self.num_correct_robot += 1
self.total_rounds_robot += 1 # it is a robot guessing here
self.total_wins_robot += 1
self._save_totals()
# Announce the robot's victory
self.robot_connection.happy_feedback()
self.robot_connection.say(fb)
# Now we can add it to the existing dataset (for future classification and generation)
if self.num_answer_attempts == 1:
self.gist_dataset.append(self.previous_recording)
self.gist_dataset.append(gist)
self.gist_dataset.save("data/gists.csv")
self._regenerate_cluster(self.current_concept['id'])
# Move on
self.concept_index += 1
self.websocket_server.send("hide_item()")
print str(self.concept_index) + " " + str(len(self.concepts_participant)) + " " + str(len(self.concepts_robot))
# Check whether we've covered all 5 items, then end the session
if self.concept_index == len(self.concepts_participant):
self._log('experiment_end(robot_score=' + str(self.num_correct_robot) + ',participant_score=' + str(self.num_correct_participant) + ')')
# Announce the end of the session
self.robot_connection.say(Speech.get(self.lang, 'outro_1'))
# Depending on who guessed most gestures correctly (this is very likely the participant),
# we have a slightly different outro
if self.num_correct_robot > self.num_correct_participant:
self.robot_connection.say(Speech.get(self.lang, 'outro_robot_wins'))
else:
self.robot_connection.say(Speech.get(self.lang, 'outro_participant_wins'))
self.robot_connection.say(Speech.get(self.lang, 'outro_2'))
# Clean up
self.websocket_server.send("hide_classification()")
self.socket_server.send("controlpanel", "ExperimentFinished()")
# Robot can rest
self.robot_connection.end_experiment()
else: # This was not the last round, so we proceed with the next turn for the robot to perform
self._do_robot_turn()
# The robot guessed incorrectly!
else:
self.total_rounds_robot += 1 # it is a robot guessing
self._save_totals()
self.robot_connection.say(Speech.get(self.lang, 'negative_feedback_self'))
self.num_answer_attempts += 1
# If this is the first attempt, try again
if self.num_answer_attempts == 1:
self.robot_connection.say(Speech.get(self.lang, 'participant_try_again'))
self.websocket_server.send("hide_classification()")
self.previous_guess = guess_obj[0][0]
self.previous_recording = gist
# The participant will be asked to perform another gesture for the same item
self._do_participant_second_turn()
# Otherwise, call it quits and move on :-)
else:
self.concept_index += 1
self.websocket_server.send("hide_item()")
# Now we can add it to the existing dataset (for future classification and generation)
self.gist_dataset.append(self.previous_recording)
self.gist_dataset.append(gist)
self.gist_dataset.save("data/gists.csv")
self._regenerate_cluster(self.current_concept['id'])
print str(self.concept_index) + " " + str(len(self.concepts_participant)) + " " + str(len(self.concepts_robot))
# Check whether we've covered all 5 items, then end the session
if self.concept_index == len(self.concepts_participant):
self.robot_connection.say(Speech.get(self.lang, 'outro_1'))
# Depending on who guessed most gestures correctly (this is very likely the participant),
# we have a slightly different outro
if self.num_correct_robot > self.num_correct_participant:
self.robot_connection.say(Speech.get(self.lang, 'outro_robot_wins'))
else:
self.robot_connection.say(Speech.get(self.lang, 'outro_participant_wins'))
self.robot_connection.say(Speech.get(self.lang, 'outro_2'))
# Clean up
self.websocket_server.send("hide_classification()")
# Robot can rest
self.robot_connection.end_experiment()
else: # This was not the last round, so we proceed with the next turn for the robot to perform
self._do_robot_turn()
def answer_given(self, answer):
print "Answer received: " + answer
# If this is a practice run, the correct answer is always 'glasses'
if self.is_practice:
self._log("practice_answer_given(correct=glasses,given=" + answer + ")")
if answer == "glasses": # Correct answer
self.robot_connection.happy_feedback()
self.robot_connection.say(Speech.get(self.lang, 'practice_robot_positive_feedback'))
else: # Incorrect answer
self.robot_connection.say(Speech.get(self.lang, 'practice_robot_negative_feedback'))
# In the practice run, regardless of whether the answer was correct,
# we always move on to the next step (second part of practice run)
self._do_participant_practice_turn()
# Correct answer -- increase weights for the performed recording
elif answer == self.current_concept['id']:
self._log("answer_given(correct=" + self.current_concept['id'] + ",given=" + answer + ")")
if self.num_answer_attempts == 0: # Got it right the first time
self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] += 1
self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] += 1
else: # Got it right on the second attempt -- less of an increase to the weights because higher chance of guessing correctly
self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] += 0.75
self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] += 0.75
fb = Speech.get(self.lang, 'positive_feedback')
fb = fb.replace('[obj]', self.current_concept['article'][self.lang] + " " + self.current_concept['description'][self.lang])
self.num_correct_participant += 1
self.total_rounds_participant += 1 # it is a participant guessing
self.total_wins_participant += 1
self._save_totals() # Keep track of the scores
self.robot_connection.happy_feedback()
self.robot_connection.say(fb)
self._do_participant_turn() # Robot turn over, on to the participant's turn
# Incorrect answer -- decrease weights for the performed recording
else:
self._log("answer_given(correct=" + self.current_concept['id'] + ",given=" + answer + ")")
self.num_answer_attempts += 1
self.total_rounds_participant += 1 # it is a participant guessing
self._save_totals()
neg_feedback = Speech.get(self.lang, 'negative_feedback')
neg_feedback = neg_feedback.replace('[obj]', Concepts.find(answer)["description"][self.lang])
self.robot_connection.say(neg_feedback)
if self.num_answer_attempts == 1:
self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] -= 1
self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] -= 1
self.robot_connection.say(Speech.get(self.lang, 'robot_try_again'))
self._do_robot_second_turn()
else:
self.clusters[self.current_concept['id']][self.current_cluster_id]["weight"] -= 0.75
self.clusters[self.current_concept['id']][self.current_cluster_id]["samples"][self.current_sample_id]["weight"] -= 0.75
correct_answer = Speech.get(self.lang, 'correct_answer')
correct_answer = correct_answer.replace('[obj]', self.current_concept["article"][self.lang] + " " + self.current_concept["description"][self.lang])
self.robot_connection.say(correct_answer)
self._do_participant_turn()
def _do_robot_practice_turn(self):
self.robot_connection.say(Speech.get(self.lang, 'practice_robot_intro'))
# This loads a recorded gesture (.csv format, from Kinect)
# and plays it back. In the practice round, we always use the same one.
importer = OurImport()
gesture = importer.load(self.data_dir + "/practice/glasses/glasses_practice.csv")
self.robot_connection.perform_gesture(gesture)
# These are the answers included only for the practice round
# If you want to add another language, don't forget to translate the
# terms here. These are not part of the recorded set of 35 items.
answer_data = dict()
answer_data["correct_answer"] = "glasses"
answer_data["answers"] = [
{
'id': 'windmill',
'description': {
'nl': 'Molen',
'en': 'Mill'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'static',
'image_filename': 'windmill.png'
},
{
'id': 'glasses',
'description': {
'nl': 'Bril',
'en': 'Glasses'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'tools',
'image_filename': 'glasses.png'
},
{
'id': 'elephant',
'description': {
'nl': 'Olifant',
'en': 'Elephant'
},
'article': {
'nl': 'Een',
'en': 'An'
},
'category': 'animate',
'image_filename': 'elephant.png'
},
{
'id': 'teapot',
'description': {
'nl': 'Theepot',
'en': 'Teapot'
},
'article': {
'nl': 'Een',
'en': 'A'
},
'category': 'tools',
'image_filename': 'teapot.png'
}
]
# Present answers to the participant on the tablet
self.websocket_server.send("show_answers(" + json.dumps(answer_data) + ")")
self._log("robot_practice_turn()")
self.robot_connection.say(Speech.get(self.lang, 'practice_robot_instruction'))
