class Model_RNN(LanguageModel):
def load_data(self):
pair_fname = '../lastfm_train_mappings.txt'
lyrics_path = '../data/lyrics/train/'
# X_train is a list of all examples. each examples is a 2-len list. each element is a list of words in lyrics.
# word_counts is a dictionary that maps
if self.config.debug:
X_train, l_train, self.word_counts, seq_len1, seq_len2, self.config.max_steps = get_data(pair_fname, lyrics_path, '../glove.6B.50d.txt', threshold_down=0, threshold_up=float('inf'), npos=100, nneg=100)
else:
X_train, l_train, self.word_counts, seq_len1, seq_len2, self.config.max_steps = get_data(pair_fname, lyrics_path, threshold_down=100, threshold_up=4000, npos=10000, nneg=10000)
self.labels_train = np.zeros((len(X_train),self.config.n_class))
self.labels_train[range(len(X_train)),l_train] = 1
x = collections.Counter(l_train)
for k in x.keys():
print 'class:', k, x[k]
print ''
self.vocab = Vocab()
self.vocab.construct(self.word_counts.keys())
self.wv = self.vocab.get_wv('../glove.6B.50d.txt')
with open('word_hist.csv', 'w') as f:
for w in self.word_counts.keys():
f.write(w+','+str(self.word_counts[w])+'\n')
self.encoded_train_1 = np.zeros((len(X_train), self.config.max_steps)) # need to handle this better.
self.encoded_train_2 = np.zeros((len(X_train), self.config.max_steps))
for i in range(len(X_train)):
self.encoded_train_1[i,:len(X_train[i][0])] = [self.vocab.encode(word) for word in X_train[i][0]]
self.encoded_train_2[i,:len(X_train[i][1])] = [self.vocab.encode(word) for word in X_train[i][1]]
self.sequence_len1 = np.array(seq_len1)
self.sequence_len2 = np.array(seq_len2)
def add_placeholders(self):
self.X1 = tf.placeholder(tf.int32, shape=(None, self.config.max_steps), name='X1')
self.X2 = tf.placeholder(tf.int32, shape=(None, self.config.max_steps), name='X2')
self.labels = tf.placeholder(tf.float32, shape=(None, self.config.n_class), name='labels')
#self.initial_state = tf.placeholder(tf.float32, shape=(None, self.config.hidden_size), name='initial_state')
self.seq_len1 = tf.placeholder(tf.int32, shape=(None), name='seq_len1') # for variable length sequences
self.seq_len2 = tf.placeholder(tf.int32, shape=(None), name='seq_len2') # for variable length sequences
def add_embedding(self):
#L = tf.get_variable('L', shape=(len(self.vocab), self.config.embed_size), dtype=tf.float32)
L = tf.Variable(tf.convert_to_tensor(self.wv, dtype=tf.float32), name='L')
#L = tf.constant(tf.convert_to_tensor(self.wvi), dtype=tf.float32, name='L')
inputs1 = tf.nn.embedding_lookup(L, self.X1) # self.X1 is batch_size x self.config.max_steps
inputs2 = tf.nn.embedding_lookup(L, self.X2) # input2 is batch_size x self.config.max_steps x self.config.embed_size
inputs1 = tf.split(1, self.config.max_steps, inputs1) # list of len self.config.max_steps where each element is batch_size x self.config.embed_size
inputs1 = [tf.squeeze(x, squeeze_dims=[1]) for x in inputs1]
inputs2 = tf.split(1, self.config.max_steps, inputs2) # list of len self.config.max_steps where each element is batch_size x self.config.embed_size
inputs2 = [tf.squeeze(x, squeeze_dims=[1]) for x in inputs2]
return inputs1, inputs2
def add_model_rnn(self, inputs1, inputs2, seq_len1, seq_len2):
#self.initial_state = tf.constant(np.zeros(()), dtype=tf.float32)
self.initial_state = tf.constant(np.zeros((self.config.batch_size,self.config.hidden_size)), dtype=tf.float32)
rnn_outputs = []
rnn_outputs1 = []
rnn_outputs2 = []
h_curr1 = self.initial_state
h_curr2 = self.initial_state
with tf.variable_scope('rnn'):
Whh = tf.get_variable('Whh', shape=(self.config.hidden_size,self.config.hidden_size), dtype=tf.float32)
Wxh = tf.get_variable('Wxh', shape=(self.config.embed_size,self.config.hidden_size), dtype=tf.float32)
b1 = tf.get_variable('bhx', shape=(4*self.config.hidden_size,), dtype=tf.float32)
for i in range(self.config.max_steps):
if self.config.batch_size==1:
if i==seq_len1[0]:
breaka
tmp = tf.matmul(h_curr1,Whh) + tf.matmul(inputs1[i],Wxh) + b1
rnn_outputs1.append(h_curr1)
for i in range(self.config.max_steps):
if self.config.batch_size==1:
if i==seq_len2[0]:
breaka
h_curr2 = tf.sigmoid(tf.matmul(h_curr2,Whh) + tf.matmul(inputs2[i],Wxh) + b1)
rnn_outputs2.append(h_curr2)
#lstm_states = [tf.concat(1, [rnn_outputs1[i], rnn_outputs2[i]]) for i in range(self.config.max_steps)]
rnn_final_states = tf.concat(1, [rnn_outputs1[-1], rnn_outputs2[-1]])
return rnn_final_states
def add_model_lstm(self, inputs1, inputs2, seq_len1, seq_len2):
#self.initial_state = tf.constant(np.zeros(()), dtype=tf.float32)
self.initial_state = tf.constant(np.zeros((self.config.batch_size,self.config.hidden_size)), dtype=tf.float32)
lstm_outputs1 = []
lstm_outputs2 = []
h_curr1 = self.initial_state
h_curr2 = self.initial_state
cell1 = self.initial_state
cell2 = self.initial_state
with tf.variable_scope('lstm'):
Whc = tf.get_variable('Whh', shape=(self.config.hidden_size,4*self.config.hidden_size), dtype=tf.float32, initializer=tf.random_normal_initializer())
Wxc = tf.get_variable('Wxh', shape=(self.config.embed_size,4*self.config.hidden_size), dtype=tf.float32, initializer=tf.random_normal_initializer())
b1 = tf.get_variable('bhx', shape=(self.config.hidden_size,), dtype=tf.float32, initializer=tf.random_normal_initializer())
for i in range(self.config.max_steps):
if self.config.batch_size==1:
if i==seq_len1[0]:
break
ifog1 = tf.matmul(h_curr1,Whc) + tf.matmul(inputs1[i],Wxc)
i1, f1, o1, g1 = tf.split(1, 4, ifog1)
i1 = tf.sigmoid(i1)
f1 = tf.sigmoid(f1)
o1 = tf.sigmoid(o1)
g1 = tf.tanh(g1)
cell1 = f1*cell1 + i1*g1
h_curr1 = o1*tf.tanh(cell1)
lstm_outputs1.append(h_curr1)
for i in range(self.config.max_steps):
if self.config.batch_size==1:
if i==seq_len2[0]:
break
ifog2 = tf.matmul(h_curr2,Whc) + tf.matmul(inputs2[i],Wxc)
i2, f2, o2, g2 = tf.split(1, 4, ifog2)
i2 = tf.sigmoid(i2)
f2 = tf.sigmoid(f2)
o2 = tf.sigmoid(o2)
g2 = tf.tanh(g2)
cell2 = f2*cell2 + i2*g2
h_curr2 = o2*tf.tanh(cell2)
lstm_outputs2.append(h_curr2)
lstm_final_states = tf.concat(1, [lstm_outputs1[-1], lstm_outputs2[-1]])
return lstm_final_states
def add_final_projections(self, rnn_final_states):
# rnn_outputs is a list of length batch_size of lengths = seq_len. Where each list element is ??. I think.
Whu = tf.get_variable('Whu', shape=(2*self.config.hidden_size,self.config.n_class), initializer=tf.random_normal_initializer())
bhu = tf.get_variable('bhu', shape=(self.config.n_class,), initializer=tf.random_normal_initializer())
final_projections = tf.matmul(rnn_final_states,Whu) + bhu # in case we stop short sequences, the rnn_state in further time_steps should be unch
return final_projections
def add_loss_op(self, y):
loss = tf.nn.softmax_cross_entropy_with_logits(y, self.labels)
loss = tf.reduce_mean(loss)
return loss
def add_training_op(self, loss):
#train_op = tf.train.AdamOptimizer(learning_rate=self.config.lr).minimize(loss)
train_op = tf.train.GradientDescentOptimizer(learning_rate=self.config.lr).minimize(loss)
return train_op
def __init__(self, config):
self.config = config
self.load_data()
self.add_placeholders()
self.inputs1, self.inputs2 = self.add_embedding()
if self.config.model=='rnn':
self.final_hidden_states = self.add_model_rnn(self.inputs1, self.inputs2, self.seq_len1, self.seq_len2)
elif self.config.model=='lstm':
self.final_hidden_states = self.add_model_lstm(self.inputs1, self.inputs2, self.seq_len1, self.seq_len2)
self.final_projections = self.add_final_projections(self.final_hidden_states)
self.loss = self.add_loss_op(self.final_projections)
self.train_step = self.add_training_op(self.loss)
self.predictions = tf.argmax(tf.nn.softmax(self.final_projections),1)
self.correct_predictions = tf.equal(self.predictions,tf.argmax(self.labels,1))
self.correct_predictions = tf.reduce_sum(tf.cast(self.correct_predictions, 'int32'))
def run_epoch(self, session, X1, X2, labels, sequence_len1, sequence_len2, train_op, verbose=10): # X and y are 2D np arrays
config = self.config
#state = tf.zeros([self.config.batch_size, self.config.hidden_size])
state = self.initial_state.eval()
data_len = np.shape(X1)[0]
index = np.arange(data_len)
np.random.shuffle(index)
n_batches = data_len // self.config.batch_size
loss = 0.0
total_loss = []
total_correct = 0
all_preds = -np.ones((data_len,))
for batch_num in range(n_batches):
x1_batch = X1[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size], :]
x2_batch = X2[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size], :]
labels_batch = labels[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size], :]
seq_len_batch1 = sequence_len1[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size]]
seq_len_batch2 = sequence_len2[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size]]
feed_dict = {self.X1: x1_batch,
self.X2: x2_batch,
self.labels: labels_batch,
self.seq_len1: seq_len_batch1,
self.seq_len2: seq_len_batch2}
#self.initial_state: state}
loss, preds, correct, final_projections, _ = session.run([self.loss, self.predictions, self.correct_predictions, self.final_projections, train_op], feed_dict=feed_dict)
#print str(batch_num)+'/'+str(n_batches)+' : '+str(final_projections[0][0])+' '+str(final_projections[0][1])
total_loss.append(loss)
total_correct += correct
all_preds[index[batch_num * self.config.batch_size : (batch_num+1) * self.config.batch_size]] = preds
if verbose and (batch_num+1)%verbose==0:
sys.stdout.write('\r{} / {} : loss = {:.4f} : train_acc = {:.2f}%'.format(batch_num+1, n_batches, np.mean(total_loss), 100.0*total_correct/((batch_num+1)*self.config.batch_size)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
return np.mean(total_loss), all_preds
