def train_model_embed(train, dev, glove, model, model_dir = 'models/curr_model', nb_epochs = 20, batch_size = 64, hs=True, ci = True):
X_dev_p, X_dev_h, y_dev = load_data.prepare_split_vec_dataset(dev, glove=glove)
word_index = load_data.WordIndex(glove)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
for e in range(nb_epochs):
print "Epoch ", e
mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
p = Progbar(len(train))
for i, train_index in mb:
if len(train_index) != batch_size:
continue
X_train_p, X_train_h , y_train = load_data.prepare_split_vec_dataset([train[k] for k in train_index], word_index.index)
padded_p = load_data.pad_sequences(X_train_p, maxlen = PREM_LEN, dim = -1, padding = 'pre')
padded_h = load_data.pad_sequences(X_train_h, maxlen = HYPO_LEN, dim = -1, padding = 'post')
data = {'premise_input': padded_p, 'embed_input': np.expand_dims(np.array(train_index), axis=1), 'output' : padded_h}
if ci:
data['class_input'] = y_train
if hs:
data['train_input'] = padded_h
data['output'] = np.ones((batch_size, HYPO_LEN, 1))
#sw = (padded_h != 0).astype(float)
#train_loss = float(model.train_on_batch(data, sample_weight={'output':sw})[0])
train_loss = float(model.train_on_batch(data)[0])
p.add(len(train_index),[('train_loss', train_loss)])
sys.stdout.write('\n')
model.save_weights(model_dir + '/model~' + str(e))
def adversarial_generator(train, gen_model, discriminator, word_index, beam_size):
batch_size, prem_len, _ = gen_model[0].inputs['premise'].input_shape
examples = batch_size / beam_size
hidden_size = gen_model[0].nodes['hypo_merge'].output_shape[2]
hypo_len = discriminator.input_shape[1]
while True:
mb = load_data.get_minibatches_idx(len(train), examples, shuffle=True)
for i, train_index in mb:
if len(train_index) != examples:
continue
orig_batch = [train[k] for k in train_index]
noise_input = np.random.normal(scale=0.11, size=(examples, 1, hidden_size))
class_indices = np.random.random_integers(0, 2, examples)
hypo_batch, probs = generative_predict_beam(gen_model, word_index, orig_batch,
noise_input, class_indices, True, hypo_len)
ad_preds = discriminator.predict_on_batch(hypo_batch)[0].flatten()
X_prem, _, _ = load_data.prepare_split_vec_dataset(orig_batch, word_index.index)
premise_batch = load_data.pad_sequences(X_prem, maxlen = prem_len, dim = -1,
padding = 'pre')
yield {'premise' : premise_batch, 'hypo' : hypo_batch, 'label': class_indices,
'sanity': ad_preds, 'gen_probs' : probs}
def generative_predict(test_model, word_index, batch, embed_indices, class_indices, batch_size = 64, prem_len = 22,
hypo_len = 12):
prem, _, _ = load_data.prepare_split_vec_dataset(batch, word_index)
padded_p = load_data.pad_sequences(prem, maxlen=prem_len, dim = -1)
core_model, premise_func, noise_func = test_model
premise = premise_func(padded_p)
noise = noise_func(embed_indices, load_data.convert_to_one_hot(class_indices, 3))
core_model.reset_states()
core_model.nodes['attention'].set_state(noise)
word_input = np.zeros((batch_size, 1))
result = []
for i in range(hypo_len):
data = {'premise' :premise,
'creative': noise,
'hypo_input': word_input,
'train_input': np.zeros((batch_size,1))}
preds = core_model.predict_on_batch(data)['output']
result.append(preds)
word_input = np.argmax(preds, axis=2)
result = np.transpose(np.array(result)[:,:,-1,:], (1,0,2))
return result
def generation_embded_test(train, glove, model, batch_size = 64, prem_len = 22, hypo_len = 12):
batch = np.arange(batch_size)
X_prem, X_hypo, _ = load_data.prepare_split_vec_dataset([train[k] for k in batch], glove)
X_p = load_data.pad_sequences(X_prem, maxlen = prem_len, dim = 50)
X_h = load_data.pad_sequences(X_hypo, maxlen = hypo_len, dim = 50)
data = {'premise_input': X_p, 'embed_input': np.expand_dims(np.array(batch), axis=1), 'output' : X_h}
return model.train_on_batch(data)
#p.add(len(X_p),[('train_loss', train_loss)])
def generation_test(train, glove, model, batch_size = 64, prem_len = 22, hypo_len = 12):
mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
p = Progbar(len(train))
for i, train_index in mb:
X_prem, X_hypo, _ = load_data.prepare_split_vec_dataset([train[k] for k in train_index], glove)
X_p = load_data.pad_sequences(X_prem, maxlen = prem_len, dim = 50)
X_h = load_data.pad_sequences(X_hypo, maxlen = hypo_len, dim = 50)
train_loss = model.train_on_batch(X_p, X_h)[0]
p.add(len(X_p),[('train_loss', train_loss)])
def generation_test(train,
glove,
model,
batch_size=64,
prem_len=22,
hypo_len=12):
mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
p = Progbar(len(train))
for i, train_index in mb:
X_prem, X_hypo, _ = load_data.prepare_split_vec_dataset(
[train[k] for k in train_index], glove)
X_p = load_data.pad_sequences(X_prem, maxlen=prem_len, dim=50)
X_h = load_data.pad_sequences(X_hypo, maxlen=hypo_len, dim=50)
train_loss = model.train_on_batch(X_p, X_h)[0]
p.add(len(X_p), [('train_loss', train_loss)])
def generation_predict_embed(model, word_index, batch, embed_indices, batch_size = 64, hs = True, class_indices = None):
prem, hypo, y = load_data.prepare_split_vec_dataset(batch, word_index)
X_p = load_data.pad_sequences(prem, maxlen=PREM_LEN, dim = -1)
data = {'premise_input': X_p, 'embed_input': embed_indices[:,None]}
if class_indices is not None:
C = load_data.convert_to_one_hot(class_indices, 3)
data['class_input'] = C
if hs:
data['train_input'] = np.zeros((batch_size, HYPO_LEN))
model_pred = model.predict_on_batch(data)
return model_pred['output']
def generation_embded_test(train,
glove,
model,
batch_size=64,
prem_len=22,
hypo_len=12):
batch = np.arange(batch_size)
X_prem, X_hypo, _ = load_data.prepare_split_vec_dataset(
[train[k] for k in batch], glove)
X_p = load_data.pad_sequences(X_prem, maxlen=prem_len, dim=50)
X_h = load_data.pad_sequences(X_hypo, maxlen=hypo_len, dim=50)
data = {
'premise_input': X_p,
'embed_input': np.expand_dims(np.array(batch), axis=1),
'output': X_h
}
return model.train_on_batch(data)
def generative_train_generator(train, word_index, batch_size = 64, prem_len = 22, hypo_len = 12):
while True:
mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
for i, train_index in mb:
if len(train_index) != batch_size:
continue
X_train_p, X_train_h , y_train = load_data.prepare_split_vec_dataset([train[k] for k in train_index],
word_index.index)
padded_p = load_data.pad_sequences(X_train_p, maxlen = prem_len, dim = -1, padding = 'pre')
padded_h = load_data.pad_sequences(X_train_h, maxlen = hypo_len, dim = -1, padding = 'post')
hypo_input = np.concatenate([np.zeros((batch_size, 1)), padded_h], axis = 1)
train_input = np.concatenate([padded_h, np.zeros((batch_size, 1))], axis = 1)
yield {'premise_input': padded_p,
'hypo_input': hypo_input,
'train_input' : train_input,
'noise_input' : np.expand_dims(train_index, axis=1),
'class_input' : y_train,
'output': np.ones((batch_size, hypo_len + 1, 1))}
def graph_train_batch(train, dev, model, glove, embed_size = 300):
P,H,y = load_data.prepare_split_vec_dataset(train[:128], glove)
padded_P = load_data.pad_sequences(P, dim = embed_size)
padded_H = load_data.pad_sequences(H, dim = embed_size)
data = {'premise_input': padded_P, 'hypo_input': padded_H, 'output' : y}
return model.train_on_batch(data)
def make_train_batch(orig_batch, word_index, hypo_len):
_, X_hypo, _ = load_data.prepare_split_vec_dataset(orig_batch, word_index.index)
return load_data.pad_sequences(X_hypo, maxlen = hypo_len, dim = -1, padding = 'post')
def t_func(example, loss, cpred, ctrue):
if not ctrue:
return False
vec_ex = load_data.prepare_split_vec_dataset([example, padding_example], wi.index, True)
pred = discriminator.predict(vec_ex[1][:1])[0][0]
return pred > ad_threshold
