def run_experiment(name, cname, rname, models, data): data_by_image = defaultdict(list) for datum in data: data_by_image[datum.image_id].append(datum) with open("experiments/%s/%s.ids.txt" % (name, cname)) as id_f, \ open("experiments/%s/%s.results.%s.txt" % (name, cname, rname), "w") as results_f: print >>results_f, "id,target,distractor,similarity,model_name,speaker_score,listener_score,description" counter = 0 for line in id_f: img1, img2, similarity = line.strip().split(",") assert img1 in data_by_image and img2 in data_by_image d1 = data_by_image[img1][0] d2 = data_by_image[img2][0] for model_name, model in models.items(): for i_sample in range(10): speaker_scores, listener_scores, samples = \ model.sample([d1], [[d2]], dropout=False, viterbi=False) parts = [ counter, img1, img2, similarity, model_name, speaker_scores[0], listener_scores[0], " ".join([WORD_INDEX.get(i) for i in samples[0][1:-1]]) ] print >>results_f, ",".join([str(s) for s in parts]) counter += 1
def demo(scenes, model, apollo_net, config): data = scenes[:config.batch_size] alt_indices = \ [np.random.choice(len(scenes), size=config.batch_size) for i_alt in range(config.alternatives)] alt_data = [[scenes[i] for i in alt] for alt in alt_indices] _, samples = model.sample(data, alt_data, dropout=False) for i in range(10): sample = samples[i] print data[i].image_id print " ".join([WORD_INDEX.get(i) for i in sample]) print
def sample(self, prefix, encoding, viterbi): net = self.apollo_net batch_size = net.blobs[encoding].shape[0] max_words = 20 l_seed = "LstmStringDecoder_%s_%s_seed" l_prev_word = "LstmStringDecoder_%s_%s_word_%d" l_word_vec = "LstmStringDecoder_%s_%s_word_vec_%d" l_concat = "LstmStringDecoder_%s_%s_concat_%d" l_lstm = "LstmStringDecoder_%s_%s_lstm_%d" l_hidden = "LstmStringDecoder_%s_%s_hidden_%d" l_mem = "LstmStringDecoder_%s_%s_mem_%d" l_output = "LstmStringDecoder_%s_%s_output_%d" l_softmax = "LstmStringDecoder_%s_%s_softmax_%d" p_word_vec = ["LstmStringDecoder_%s_word_vec" % self.name] p_lstm = [ "LstmStringDecoder_%s_lstm_iv" % self.name, "LstmStringDecoder_%s_lstm_ig" % self.name, "LstmStringDecoder_%s_lstm_fg" % self.name, "LstmStringDecoder_%s_lstm_og" % self.name ] p_output = [ "LstmStringDecoder_%s_output_weight" % self.name, "LstmStringDecoder_%s_output_bias" % self.name ] samples = np.zeros((batch_size, max_words), dtype=int) samples[:, 0] = WORD_INDEX["<s>"] net.f( NumpyData(l_seed, np.zeros((batch_size, self.config.hidden_size)))) for i_step in range(1, max_words): l_prev_word_i = l_prev_word % (self.name, prefix, i_step) l_word_vec_i = l_word_vec % (self.name, prefix, i_step) l_concat_i = l_concat % (self.name, prefix, i_step) l_lstm_i = l_lstm % (self.name, prefix, i_step) l_hidden_i = l_hidden % (self.name, prefix, i_step) l_mem_i = l_mem % (self.name, prefix, i_step) l_output_i = l_output % (self.name, prefix, i_step) l_softmax_i = l_softmax % (self.name, prefix, i_step) if i_step == 1: prev_hidden = l_seed prev_mem = l_seed else: prev_hidden = l_hidden % (self.name, prefix, i_step - 1) prev_mem = l_mem % (self.name, prefix, i_step - 1) net.f(NumpyData(l_prev_word_i, samples[:, i_step - 1])) net.f( Wordvec(l_word_vec_i, self.config.word_embedding_size, len(WORD_INDEX), bottoms=[l_prev_word_i], param_names=p_word_vec)) net.f( Concat(l_concat_i, bottoms=[prev_hidden, l_word_vec_i, encoding])) net.f( LstmUnit(l_lstm_i, bottoms=[l_concat_i, prev_mem], param_names=p_lstm, tops=[l_hidden_i, l_mem_i], num_cells=self.config.hidden_size)) net.f( InnerProduct(l_output_i, len(WORD_INDEX), bottoms=[l_hidden_i], param_names=p_output)) net.f(Softmax(l_softmax_i, bottoms=[l_output_i])) choices = [] for i in range(batch_size): probs = net.blobs[l_softmax_i].data[i, :].astype(np.float64) probs /= probs.sum() if viterbi: choices.append(np.argmax(probs)) else: choices.append(np.random.choice(len(WORD_INDEX), p=probs)) samples[:, i_step] = choices out_samples = [] for i in range(samples.shape[0]): this_sample = [] for j in range(samples.shape[1]): word = WORD_INDEX.get(samples[i, j]) this_sample.append(samples[i, j]) if word == "</s>": break out_samples.append(this_sample) return out_samples
def sample(self, prefix, encoding, viterbi): net = self.apollo_net max_words = 20 batch_size = net.blobs[encoding].shape[0] out_logprobs = np.zeros((batch_size, )) samples = np.zeros((batch_size, max_words)) history_features = np.zeros((batch_size, len(WORD_INDEX))) last_features = np.zeros((batch_size, len(WORD_INDEX))) samples[:, 0] = WORD_INDEX["<s>"] last_features[:, WORD_INDEX["<s>"]] += 1 l_history_data = "MlpStringDecoder_%s_%s_history_data_%d" l_last_data = "MlpStringDecoder_%s_%s_last_data_%d" l_cat_features = "MlpStringDecoder_%s_%s_cat_features_%d" l_ip1 = "MlpStringDecoder_%s_%s_ip1_%d" l_cat = "MlpStringDecoder_%s_%s_cat_%d" l_relu1 = "MlpStringDecoder_%s_%s_relu1_%d" l_ip2 = "MlpStringDecoder_%s_%s_ip2_%d" l_softmax = "MlpStringDecoder_%s_%s_softmax_%d" p_ip1 = [ "MlpStringDecoder_%s_ip1_weight" % self.name, "MlpStringDecoder_%s_ip1_bias" % self.name ] p_ip2 = [ "MlpStringDecoder_%s_ip2_weight" % self.name, "MlpStringDecoder_%s_ip2_bias" % self.name ] for i_step in range(1, max_words): l_history_data_i = l_history_data % (self.name, prefix, i_step) l_last_data_i = l_last_data % (self.name, prefix, i_step) l_cat_features_i = l_cat_features % (self.name, prefix, i_step) l_ip1_i = l_ip1 % (self.name, prefix, i_step) l_cat_i = l_cat % (self.name, prefix, i_step) l_relu1_i = l_relu1 % (self.name, prefix, i_step) l_ip2_i = l_ip2 % (self.name, prefix, i_step) l_softmax_i = l_softmax % (self.name, prefix, i_step) net.f(DummyData(l_history_data_i, (1, 1, 1, 1))) net.blobs[l_history_data_i].reshape(history_features.shape) net.f(DummyData(l_last_data_i, (1, 1, 1, 1))) net.blobs[l_last_data_i].reshape(last_features.shape) net.blobs[l_history_data_i].data[...] = history_features net.blobs[l_last_data_i].data[...] = last_features net.f( Concat(l_cat_features_i, bottoms=[l_history_data_i, l_last_data_i])) net.f( InnerProduct(l_ip1_i, self.config.hidden_size, bottoms=[l_cat_features_i], param_names=p_ip1)) net.f(Concat(l_cat_i, bottoms=[l_ip1_i, encoding])) net.f(ReLU(l_relu1_i, bottoms=[l_cat_i])) net.f( InnerProduct(l_ip2_i, len(WORD_INDEX), bottoms=[l_relu1_i], param_names=p_ip2)) net.f(Softmax(l_softmax_i, bottoms=[l_ip2_i])) probs = net.blobs[l_softmax_i].data history_features += last_features last_features[...] = 0 for i_datum in range(batch_size): d_probs = probs[i_datum, :].astype(float) d_probs /= d_probs.sum() if viterbi: choice = d_probs.argmax() else: choice = np.random.multinomial(1, d_probs).argmax() samples[i_datum, i_step] = choice last_features[i_datum, choice] += 1 out_logprobs[i_datum] += np.log(d_probs[choice]) out_samples = [] for i in range(samples.shape[0]): this_sample = [] for j in range(samples.shape[1]): word = WORD_INDEX.get(samples[i, j]) #this_sample.append(word) this_sample.append(samples[i, j]) if word == "</s>": break if this_sample[-1] != WORD_INDEX["</s>"]: this_sample.append(WORD_INDEX["</s>"]) out_samples.append(this_sample) return out_logprobs, out_samples
def sample(self, prefix, encoding, viterbi): net = self.apollo_net batch_size = net.blobs[encoding].shape[0] max_words = 20 l_seed = "LstmStringDecoder_%s_%s_seed" l_prev_word = "LstmStringDecoder_%s_%s_word_%d" l_word_vec = "LstmStringDecoder_%s_%s_word_vec_%d" l_concat = "LstmStringDecoder_%s_%s_concat_%d" l_lstm = "LstmStringDecoder_%s_%s_lstm_%d" l_hidden = "LstmStringDecoder_%s_%s_hidden_%d" l_mem = "LstmStringDecoder_%s_%s_mem_%d" l_output = "LstmStringDecoder_%s_%s_output_%d" l_softmax = "LstmStringDecoder_%s_%s_softmax_%d" p_word_vec = ["LstmStringDecoder_%s_word_vec" % self.name] p_lstm = ["LstmStringDecoder_%s_lstm_iv" % self.name, "LstmStringDecoder_%s_lstm_ig" % self.name, "LstmStringDecoder_%s_lstm_fg" % self.name, "LstmStringDecoder_%s_lstm_og" % self.name] p_output = ["LstmStringDecoder_%s_output_weight" % self.name, "LstmStringDecoder_%s_output_bias" % self.name] samples = np.zeros((batch_size, max_words), dtype=int) samples[:,0] = WORD_INDEX["<s>"] net.f(NumpyData( l_seed, np.zeros((batch_size, self.config.hidden_size)))) for i_step in range(1, max_words): l_prev_word_i = l_prev_word % (self.name, prefix, i_step) l_word_vec_i = l_word_vec % (self.name, prefix, i_step) l_concat_i = l_concat % (self.name, prefix, i_step) l_lstm_i = l_lstm % (self.name, prefix, i_step) l_hidden_i = l_hidden % (self.name, prefix, i_step) l_mem_i = l_mem % (self.name, prefix, i_step) l_output_i = l_output % (self.name, prefix, i_step) l_softmax_i = l_softmax % (self.name, prefix, i_step) if i_step == 1: prev_hidden = l_seed prev_mem = l_seed else: prev_hidden = l_hidden % (self.name, prefix, i_step - 1) prev_mem = l_mem % (self.name, prefix, i_step - 1) net.f(NumpyData(l_prev_word_i, samples[:, i_step-1])) net.f(Wordvec( l_word_vec_i, self.config.word_embedding_size, len(WORD_INDEX), bottoms=[l_prev_word_i], param_names=p_word_vec)) net.f(Concat( l_concat_i, bottoms=[prev_hidden, l_word_vec_i, encoding])) net.f(LstmUnit( l_lstm_i, bottoms=[l_concat_i, prev_mem], param_names=p_lstm, tops=[l_hidden_i, l_mem_i], num_cells=self.config.hidden_size)) net.f(InnerProduct( l_output_i, len(WORD_INDEX), bottoms=[l_hidden_i], param_names=p_output)) net.f(Softmax(l_softmax_i, bottoms=[l_output_i])) choices = [] for i in range(batch_size): probs = net.blobs[l_softmax_i].data[i,:].astype(np.float64) probs /= probs.sum() if viterbi: choices.append(np.argmax(probs)) else: choices.append(np.random.choice(len(WORD_INDEX), p=probs)) samples[:, i_step] = choices out_samples = [] for i in range(samples.shape[0]): this_sample = [] for j in range(samples.shape[1]): word = WORD_INDEX.get(samples[i,j]) this_sample.append(samples[i,j]) if word == "</s>": break out_samples.append(this_sample) return out_samples
def sample(self, prefix, encoding, viterbi): net = self.apollo_net max_words = 20 batch_size = net.blobs[encoding].shape[0] out_logprobs = np.zeros((batch_size,)) samples = np.zeros((batch_size, max_words)) history_features = np.zeros((batch_size, len(WORD_INDEX))) last_features = np.zeros((batch_size, len(WORD_INDEX))) samples[:,0] = WORD_INDEX["<s>"] last_features[:,WORD_INDEX["<s>"]] += 1 l_history_data = "MlpStringDecoder_%s_%s_history_data_%d" l_last_data = "MlpStringDecoder_%s_%s_last_data_%d" l_cat_features = "MlpStringDecoder_%s_%s_cat_features_%d" l_ip1 = "MlpStringDecoder_%s_%s_ip1_%d" l_cat = "MlpStringDecoder_%s_%s_cat_%d" l_relu1 = "MlpStringDecoder_%s_%s_relu1_%d" l_ip2 = "MlpStringDecoder_%s_%s_ip2_%d" l_softmax = "MlpStringDecoder_%s_%s_softmax_%d" p_ip1 = ["MlpStringDecoder_%s_ip1_weight" % self.name, "MlpStringDecoder_%s_ip1_bias" % self.name] p_ip2 = ["MlpStringDecoder_%s_ip2_weight" % self.name, "MlpStringDecoder_%s_ip2_bias" % self.name] for i_step in range(1, max_words): l_history_data_i = l_history_data % (self.name, prefix, i_step) l_last_data_i = l_last_data % (self.name, prefix, i_step) l_cat_features_i = l_cat_features % (self.name, prefix, i_step) l_ip1_i = l_ip1 % (self.name, prefix, i_step) l_cat_i = l_cat % (self.name, prefix, i_step) l_relu1_i = l_relu1 % (self.name, prefix, i_step) l_ip2_i = l_ip2 % (self.name, prefix, i_step) l_softmax_i = l_softmax % (self.name, prefix, i_step) net.f(DummyData(l_history_data_i, (1,1,1,1))) net.blobs[l_history_data_i].reshape(history_features.shape) net.f(DummyData(l_last_data_i, (1,1,1,1))) net.blobs[l_last_data_i].reshape(last_features.shape) net.blobs[l_history_data_i].data[...] = history_features net.blobs[l_last_data_i].data[...] = last_features net.f(Concat(l_cat_features_i, bottoms=[l_history_data_i, l_last_data_i])) net.f(InnerProduct( l_ip1_i, self.config.hidden_size, bottoms=[l_cat_features_i], param_names=p_ip1)) net.f(Concat(l_cat_i, bottoms=[l_ip1_i, encoding])) net.f(ReLU(l_relu1_i, bottoms=[l_cat_i])) net.f(InnerProduct( l_ip2_i, len(WORD_INDEX), bottoms=[l_relu1_i], param_names=p_ip2)) net.f(Softmax(l_softmax_i, bottoms=[l_ip2_i])) probs = net.blobs[l_softmax_i].data history_features += last_features last_features[...] = 0 for i_datum in range(batch_size): d_probs = probs[i_datum,:].astype(float) d_probs /= d_probs.sum() if viterbi: choice = d_probs.argmax() else: choice = np.random.multinomial(1, d_probs).argmax() samples[i_datum, i_step] = choice last_features[i_datum, choice] += 1 out_logprobs[i_datum] += np.log(d_probs[choice]) out_samples = [] for i in range(samples.shape[0]): this_sample = [] for j in range(samples.shape[1]): word = WORD_INDEX.get(samples[i,j]) #this_sample.append(word) this_sample.append(samples[i,j]) if word == "</s>": break if this_sample[-1] != WORD_INDEX["</s>"]: this_sample.append(WORD_INDEX["</s>"]) out_samples.append(this_sample) return out_logprobs, out_samples