def process(self, sess, iterator, mode, optimizer=list(), store_games=False): # initialize the wrapper self.qgen.initialize(sess) self.oracle.initialize(sess) self.guesser.initialize(sess) self.storage = [] score, total_elem = 0, 0 for game_data in tqdm(iterator): # initialize the dialogue full_dialogues = [ np.array([self.tokenizer.start_token]) for _ in range(self.batch_size) ] prev_answers = full_dialogues prob_objects = [] no_elem = len(game_data["raw"]) total_elem += no_elem # Step 1: generate question/answer self.qgen.reset(batch_size=no_elem) for no_question in range(self.max_no_question): # Step 1.1: Generate new question padded_questions, questions, seq_length = \ self.qgen.sample_next_question(sess, prev_answers, game_data=game_data, mode=mode) # Step 1.2: Answer the question answers = self.oracle.answer_question( sess, question=padded_questions, seq_length=seq_length, game_data=game_data) # Step 1.3: store the full dialogues for i in range(self.batch_size): full_dialogues[i] = np.concatenate( (full_dialogues[i], questions[i], [answers[i]])) # Step 1.4 set new input tokens prev_answers = [[a] for a in answers] # Step 1.5 Compute the probability of finding the object after each turn if store_games: padded_dialogue, seq_length = list_to_padded_tokens( full_dialogues, self.tokenizer) _, softmax, _ = self.guesser.find_object( sess, padded_dialogue, seq_length, game_data) prob_objects.append(softmax) # Step 1.6 Check if all dialogues are stopped has_stop = True for d in full_dialogues: has_stop &= self.tokenizer.stop_dialogue in d if has_stop: break # Step 2 : clear question after <stop_dialogue> full_dialogues, _ = clear_after_stop_dialogue( full_dialogues, self.tokenizer) padded_dialogue, seq_length = list_to_padded_tokens( full_dialogues, self.tokenizer) # Step 3 : Find the object found_object, _, id_guess_objects = self.guesser.find_object( sess, padded_dialogue, seq_length, game_data) score += np.sum(found_object) nb_badwords = [] for i, d in enumerate(full_dialogues): nb_d = 0 for w in d: if (w in self.avoid_words): nb_d += 1 nb_badwords.append(nb_d) if store_games: prob_objects = np.transpose(prob_objects, axes=[1, 0, 2]) for i, (d, g, t, f, go, po) in enumerate( zip(full_dialogues, game_data["raw"], game_data["targets_index"], found_object, id_guess_objects, prob_objects)): self.storage.append({ "dialogue": d, "game": g, "object_id": g.objects[t].id, "success": f, "guess_object_id": g.objects[go].id, "prob_objects": po }) if len(optimizer) > 0: final_reward = [ int(found_object[i]) - nb_badwords[i] * self.pen_spatial for i in range(len(nb_badwords)) ] # + 0 # +1 if found otherwise 0 self.apply_policy_gradient(sess, final_reward=final_reward, padded_dialogue=padded_dialogue, seq_length=seq_length, game_data=game_data, optimizer=optimizer) score = 1.0 * score / iterator.n_examples return score
def process(self, sess, iterator, mode, optimizer=list(), store_games=False): # initialize the wrapper self.qgen.initialize(sess) self.oracle.initialize(sess) self.guesser.initialize(sess) self.storage = [] score, total_elem = 0, 0 for game_data in tqdm(iterator): # initialize the dialogue full_dialogues = [np.array([self.tokenizer.start_token]) for _ in range(self.batch_size)] prev_answers = full_dialogues no_elem = len(game_data["raw"]) total_elem += no_elem # Step 1: generate question/answer self.qgen.reset(batch_size=no_elem) for no_question in range(self.max_no_question): # Step 1.1: Generate new question padded_questions, questions, seq_length = \ self.qgen.sample_next_question(sess, prev_answers, game_data=game_data, mode=mode) # Step 1.2: Answer the question answers = self.oracle.answer_question(sess, question=padded_questions, seq_length=seq_length, game_data=game_data) # Step 1.3: store the full dialogues for i in range(self.batch_size): full_dialogues[i] = np.concatenate((full_dialogues[i], questions[i], [answers[i]])) # Step 1.4 set new input tokens prev_answers = [[a]for a in answers] # Step 2 : clear question after <stop_dialogue> full_dialogues, _ = clear_after_stop_dialogue(full_dialogues, self.tokenizer) padded_dialogue, seq_length = list_to_padded_tokens(full_dialogues, self.tokenizer) # Step 3 : Find the object found_object, softmax, guess_objects = self.guesser.find_object(sess, padded_dialogue, seq_length, game_data) score += np.sum(found_object) if store_games: for d, g, t, f, go in zip(full_dialogues, game_data["raw"], game_data["targets"], found_object, guess_objects): self.storage.append({"dialogue": d, "game": g, "object_id": g.objects[t].id, "success": f, "guess_object_id": g.objects[go].id}) if len(optimizer) > 0: final_reward = found_object + 0 # +1 if found otherwise 0 self.apply_policy_gradient(sess, final_reward=final_reward, padded_dialogue=padded_dialogue, seq_length=seq_length, game_data=game_data, optimizer=optimizer) score = 1.0 * score / iterator.n_examples return score
def process(self, sess, iterator, mode, optimizer=list(), store_games=False): # initialize the wrapper self.qgen.initialize(sess) self.oracle.initialize(sess) self.guesser.initialize(sess) self.storage = [] score, total_elem = 0, 0 ### Here we eventually modify the word_counter, that will modify the reward function self.current_batch+=1 if (self.current_batch % self.update_reward) ==0: for w in self.current_word_counter: self.word_counter[w]+=self.current_word_counter[w] self.current_word_counter=collections.Counter() print("<----reward upadted---->") for game_data in tqdm(iterator): # initialize the dialogue full_dialogues = [np.array([self.tokenizer.start_token]) for _ in range(self.batch_size)] prev_answers = full_dialogues prob_objects = [] no_elem = len(game_data["raw"]) total_elem += no_elem # Step 1: generate question/answer self.qgen.reset(batch_size=no_elem) for no_question in range(self.max_no_question): # Step 1.1: Generate new question padded_questions, questions, seq_length = \ self.qgen.sample_next_question(sess, prev_answers, game_data=game_data, mode=mode) # Step 1.2: Answer the question answers = self.oracle.answer_question(sess, question=padded_questions, seq_length=seq_length, game_data=game_data) # Step 1.3: store the full dialogues for i in range(self.batch_size): full_dialogues[i] = np.concatenate((full_dialogues[i], questions[i], [answers[i]])) # Step 1.4 set new input tokens prev_answers = [[a]for a in answers] # Step 1.5 Compute the probability of finding the object after each turn if store_games: padded_dialogue, seq_length = list_to_padded_tokens(full_dialogues, self.tokenizer) _, softmax, _ = self.guesser.find_object(sess, padded_dialogue, seq_length, game_data) prob_objects.append(softmax) # Step 1.6 Check if all dialogues are stopped has_stop = True for d in full_dialogues: has_stop &= self.tokenizer.stop_dialogue in d if has_stop: break # Step 2 : clear question after <stop_dialogue> full_dialogues, _ = clear_after_stop_dialogue(full_dialogues, self.tokenizer) padded_dialogue, seq_length = list_to_padded_tokens(full_dialogues, self.tokenizer) # Step 3 : Find the object found_object, _, id_guess_objects = self.guesser.find_object(sess, padded_dialogue, seq_length, game_data) score += np.sum(found_object) exploration_reward=[] repetitions=[] for i, d in enumerate(full_dialogues) : occs=collections.Counter() exploration_bonus=0.0 n_words=0 for w in d: n_words+=1 self.current_word_counter[w]+=1 if not w in self.common_words: occs[w]+=1 exploration_bonus+=1.0/(1.0*(self.word_counter[w]+1)) exploration_bonus/=(1.0*(n_words+1.0)) exploration_reward.append(exploration_bonus) repetition=0 for w in occs: repetition+=(occs[w]-1) repetitions.append(repetition) if store_games: prob_objects = np.transpose(prob_objects, axes=[1,0,2]) for i, (d, g, t, f, go, po) in enumerate(zip(full_dialogues, game_data["raw"], game_data["targets_index"], found_object, id_guess_objects, prob_objects)): self.storage.append({"dialogue": d, "game": g, "object_id": g.objects[t].id, "success": f, "guess_object_id": g.objects[go].id, "prob_objects" : po} ) if len(optimizer) > 0: final_reward = [(int(found_object[i])+exploration_reward[i]*self.bonus_expl - repetitions[i]*self.penalty_repetition) for i in range(len(exploration_reward))] self.apply_policy_gradient(sess, final_reward=final_reward, padded_dialogue=padded_dialogue, seq_length=seq_length, game_data=game_data, optimizer=optimizer) score = 1.0 * score / iterator.n_examples return score