def trainIters(vocab, embedding, smn, optimizer, train_examples, dev_examples, n_iters, learning_rate, batch_size,
infer_batch_size, print_every=1000, plot_every=100):
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
best_score = 0.0
criterion = nn.NLLLoss()
# 在有Model时预先生成best_score
print("============evaluate_start==================")
score = evaluate(embedding, smn, dev_examples, infer_batch_size)
print('R10@1_socre: {0}'.format(score))
if score >= best_score:
best_score = score
torch.save(smn.state_dict(), model_dir + '/smn')
torch.save(embedding.state_dict(), model_dir + '/embedding')
torch.save(optimizer.state_dict(), model_dir + '/optimizer')
print('new ' + model_dir + ' saved.')
print("==============evaluate_end==================")
for iter in range(1, n_iters + 1):
print("======================iter%s============================" % iter)
for i, (utterances, response, label) in enumerate(get_minibatches(train_examples, batch_size)):
# print("batch: ", i)
# 构建负例（1:1）（实际训练数据大小为batch * 2）
neg_response = []
for idx in range(len(label)):
neg = np.random.randint(0, len(train_examples[1]))
neg_response.append(train_examples[1][neg])
utterances = torch.tensor(utterances + utterances, dtype=torch.long, device=device_cuda)
response = torch.tensor(response + neg_response, dtype=torch.long, device=device_cuda)
label = torch.tensor(label + [0] * len(label), dtype=torch.long, device=device_cuda)
loss = train(utterances, response, label, embedding, smn, optimizer, criterion)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('(%d %d%%) %.4f' % (iter, iter / n_iters * 100, print_loss_avg))
if iter % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
print("============evaluate_start==================")
score = evaluate(embedding, smn, dev_examples, infer_batch_size)
print('R10@1_socre: {0}'.format(score))
if score >= best_score:
best_score = score
torch.save(smn.state_dict(), model_dir + '/smn')
torch.save(embedding.state_dict(), model_dir + '/embedding')
torch.save(optimizer.state_dict(), model_dir + '/optimizer')
print('new ' + model_dir + ' saved.')
print("==============evaluate_end==================")
def run_epoch(self, sess, inputs, labels):
n_minibatches, total_loss = 0, 0
for input_batch, labels_batch in get_minibatches(
[inputs, labels], self.config.batch_size):
n_minibatches += 1
total_loss = self.train_on_batch(sess, input_batch, labels_batch)
return total_loss / n_minibatches
def predict_fine(self, x_xyz, ray_dir=None, chunksize=8192):
x = self.embed_xyz_fine(x_xyz.reshape((-1, x_xyz.shape[-1])))
if self.embed_dir_fine is not None:
x_dir = ray_dir / ray_dir.norm(p=2, dim=-1).unsqueeze(-1)
x_dir = x_dir[..., None, :].expand(x_xyz.shape)
x_dir = self.embed_dir_fine(x_dir.reshape((-1, x_dir.shape[-1])))
x = torch.cat((x, x_dir), dim=-1)
batches = get_minibatches(x, chunksize)
out = torch.cat([self.model_fine(xin) for xin in batches], dim=0)
return out.reshape(x_xyz.shape[:-1] + out.shape[-1:])
def gen_examples(data, mask, batch_size, concat=False):
"""
Divide examples into batches of size `batch_size`.
"""
minibatches = utils.get_minibatches(len(data[0]), batch_size)
all_ex = []
for minibatch in minibatches:
dm = []
for d in data:
dm += [d[minibatch]]
for m in mask:
dm += [m[minibatch]]
all_ex += [dm]
return all_ex
def gen_examples(x1, x2, l, y, batch_size):
"""
Divide examples into batches of size `batch_size`.
"""
minibatches = utils.get_minibatches(len(x1), batch_size)
all_ex = []
for minibatch in minibatches:
mb_x1 = [x1[t] for t in minibatch]
mb_x2 = [x2[t] for t in minibatch]
mb_l = l[minibatch]
mb_y = [y[t] for t in minibatch]
mb_x1, mb_mask1 = utils.prepare_data(mb_x1)
mb_x2, mb_mask2 = utils.prepare_data(mb_x2)
all_ex.append((mb_x1, mb_mask1, mb_x2, mb_mask2, mb_l, mb_y))
return all_ex
def gen_examples(x1, x2, l, y, batch_size):
"""
Divide examples into batches of size `batch_size`.
"""
minibatches = utils.get_minibatches(len(x1), batch_size)
all_ex = []
for minibatch in minibatches:
mb_x1 = [x1[t] for t in minibatch]
mb_x2 = [x2[t] for t in minibatch]
mb_l = l[minibatch]
mb_y = [y[t] for t in minibatch]
mb_x1, mb_mask1 = utils.prepare_data(mb_x1)
mb_x2, mb_mask2 = utils.prepare_data(mb_x2)
all_ex.append((mb_x1, mb_mask1, mb_x2, mb_mask2, mb_l, mb_y))
return all_ex
def gen_examples(x1, x2, x3, y, batch_size, concat=False):
"""
Divide examples into batches of size `batch_size`.
"""
minibatches = utils.get_minibatches(len(x1), batch_size)
all_ex = []
for minibatch in minibatches:
mb_x1 = [x1[t] for t in minibatch]
mb_x2 = [x2[t] for t in minibatch]
mb_x3 = [x3[t * 4 + k] for t in minibatch for k in range(4)]
mb_y = [y[t] for t in minibatch]
mb_x1, mb_mask1 = utils.prepare_data(mb_x1)
mb_x2, mb_mask2 = utils.prepare_data(mb_x2)
mb_x3, mb_mask3 = utils.prepare_data(mb_x3)
all_ex.append((mb_x1, mb_mask1, mb_x2, mb_mask2, mb_x3, mb_mask3, mb_y))
return all_ex
def gen_examples(articles,
questions,
options,
answers,
labels,
batch_size,
concat=False):
"""
Divide examples into batches of size `batch_size`.
"""
minibatches = utils.get_minibatches(len(articles), batch_size)
all_ex = []
for minibatch in minibatches:
if len(minibatch) < batch_size:
break
mb_art = [articles[t] for t in minibatch for k in range(4)]
mb_que = [questions[t] for t in minibatch for k in range(4)]
mb_opt = [options[t * 4 + k] for t in minibatch for k in range(4)]
mb_ans = [answers[t] for t in minibatch for k in range(4)]
mb_labels = [labels[t * 4 + k] for t in minibatch for k in range(4)]
all_ex.append((mb_art, mb_que, mb_opt, mb_ans, mb_labels))
return all_ex
def trainIters(lang,
embedding,
encoder,
decoder,
optimizer,
train_pairs,
dev_pairs,
max_length,
n_iters,
learning_rate,
batch_size,
infer_batch_size,
print_every=1000,
plot_every=100):
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
best_bleu_score = 0.0
criterion = nn.NLLLoss(ignore_index=0)
# 在有Model时预先生成best_bleu_score
print("============evaluate_start==================")
bleu_score = evaluate(lang, embedding, encoder, decoder, dev_pairs,
max_length, infer_batch_size)
print('bleu_socre: {0}'.format(bleu_score))
if bleu_score >= best_bleu_score:
best_bleu_score = bleu_score
torch.save(encoder.state_dict(), model_dir + '/encoder')
torch.save(decoder.state_dict(), model_dir + '/decoder')
torch.save(embedding.state_dict(), model_dir + '/embedding')
torch.save(optimizer.state_dict(), model_dir + '/optimizer')
print('new ' + model_dir + ' saved.')
print("==============evaluate_end==================")
for iter in range(1, n_iters + 1):
print("======================iter%s============================" %
iter)
for i, training_batch in enumerate(
get_minibatches(train_pairs, batch_size)):
# print("batch: ", i)
# 排序并padding
training_batch = sorted(training_batch,
key=lambda x: len(x[0]),
reverse=True)
training_batch = list(
map(lambda x: indexesFromPair(lang, x), training_batch))
enc_lens = list(map(lambda x: len(x[0]), training_batch))
dec_lens = list(map(lambda x: len(x[1]), training_batch))
# print("enc_lens: ", enc_lens)
# print("dec_lens: ", dec_lens)
enc_max_len = max(enc_lens)
dec_max_len = max(dec_lens)
enc = []
dec = []
for t in training_batch:
enc.append(t[0] + (enc_max_len - len(t[0])) * [0])
dec.append(t[1] + (dec_max_len - len(t[1])) * [0])
enc = torch.tensor(enc, dtype=torch.long, device=device_cuda)
dec = torch.tensor(dec, dtype=torch.long, device=device_cuda)
loss = train(enc, enc_lens, dec, dec_lens, embedding, encoder,
decoder, optimizer, criterion)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('(%d %d%%) %.4f' %
(iter, iter / n_iters * 100, print_loss_avg))
if iter % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
print("============evaluate_start==================")
bleu_score = evaluate(lang, embedding, encoder, decoder, dev_pairs,
max_length, infer_batch_size)
print('bleu_socre: {0}'.format(bleu_score))
if bleu_score >= best_bleu_score:
best_bleu_score = bleu_score
torch.save(encoder.state_dict(), model_dir + '/encoder')
torch.save(decoder.state_dict(), model_dir + '/decoder')
torch.save(embedding.state_dict(), model_dir + '/embedding')
torch.save(optimizer.state_dict(), model_dir + '/optimizer')
print('new ' + model_dir + ' saved.')
print("==============evaluate_end==================")
def evaluate(embed, devData, config):
# Create input placeholders
comments = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
masks = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentps = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
maskps = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentfs = tf.placeholder(tf.float32, [None, config["numCommentfs"]])
labels = tf.placeholder(tf.float32, [None, config["numClasses"]])
dropoutKeepProb = tf.placeholder(tf.float32)
# Create embedding tranform.
with tf.variable_scope("embedding"):
E = tf.get_variable("E", initializer=embed, trainable=False)
embeddings = tf.expand_dims(tf.nn.embedding_lookup(E, comments), -1)
embeddingps = tf.expand_dims(tf.nn.embedding_lookup(E, commentps), -1)
# CNN
cnnOutputs = getCNNOutputs(embeddings, dropoutKeepProb, "c", config)
numFiltersTotal = config["numFilters"] * len(config["filterSizes"])
if config["addCommentp"]:
cnnOutputps = getCNNOutputs(embeddingps, dropoutKeepProb, "p", config)
S = tf.get_variable(
"S",
shape=[numFiltersTotal, numFiltersTotal],
initializer=tf.initializers.truncated_normal(stddev=0.1))
cnnOutputps = tf.expand_dims(tf.reduce_sum(tf.matmul(cnnOutputs, S) * cnnOutputps, axis=1), -1)
cnnOutputs = tf.concat([cnnOutputs, cnnOutputps], axis=1)
numFiltersTotal += 1
# Dropout
hDroutput = tf.nn.dropout(cnnOutputs, dropoutKeepProb)
# Add in extra features
if config["addCommentf"]:
hDroutput = tf.concat([hDroutput, commentfs], axis=1)
# Layer 1 ReLu
with tf.variable_scope("layer1"):
W1 = tf.get_variable(
"W1",
shape=[numFiltersTotal + config["numCommentfs"], config["layer2Units"]],
initializer=tf.initializers.truncated_normal(stddev=0.1))
b1 = tf.get_variable(
"bias1",
shape=[config["layer2Units"]],
initializer=tf.constant_initializer(0.1))
layer1Output = tf.nn.relu(tf.matmul(hDroutput, W1) + b1)
# Dropout
layer1Droutput = tf.nn.dropout(layer1Output, dropoutKeepProb)
# Output
with tf.variable_scope("layer2"):
W2 = tf.get_variable(
"W2",
shape=[config["layer2Units"], config["numClasses"]],
initializer=tf.initializers.truncated_normal(stddev=0.1))
b2 = tf.get_variable(
"bias2",
shape=[config["numClasses"]],
initializer=tf.constant_initializer(0.1))
prediction = tf.matmul(layer1Droutput, W2) + b2
soft = tf.nn.softmax(prediction)
# Accuracy
prediction2 = tf.argmax(prediction, 1)
labels2 = tf.argmax(labels, 1)
confusionMatrix = tf.confusion_matrix(labels2, prediction2)
accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction2, labels2), tf.float32))
# Saver
saver = tf.train.Saver()
with tf.Session() as sess:
# Variable Initialization.
if config["restoreDir"]:
print "Restoring from {}".format(config["restoreDir"])
saver.restore(sess, config["restoreDir"])
else:
sess.run(tf.global_variables_initializer())
# Evaluate on test
epochPredictions = []
epochAccuracy = 0
epochConfusionMatrix = 0
totalNum = 0
for batchNum, batches in enumerate(utils.get_minibatches(devData, config["batchSize"], False)):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
dropoutKeepProb: 1.0
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchSize = len(batches[0])
batchPredictions, batchConfusionMatrix, batchAccuracy = sess.run([soft, confusionMatrix, accuracy], feedDict)
epochPredictions.extend([pred.tolist() for pred in batchPredictions])
epochConfusionMatrix += np.asarray(batchConfusionMatrix)
epochAccuracy += batchAccuracy * batchSize
totalNum += batchSize
if (batchNum + 1) % 100 == 0:
print "Batch: {}".format(batchNum + 1)
precision = epochConfusionMatrix[1][1] / float(epochConfusionMatrix[0][1] + epochConfusionMatrix[1][1])
recall = epochConfusionMatrix[1][1] / float(epochConfusionMatrix[1][0] + epochConfusionMatrix[1][1])
print "Test Accuracy: {0:.4f} Precision: {1:.4f} Recall: {2:.4f} F1: {3:.4f}".format(
epochAccuracy / float(totalNum),
precision,
recall,
2*precision*recall/(precision + recall))
print epochConfusionMatrix
# Return series.
return epochPredictions
def train(embed, trainData, devData, config):
# Create input placeholders
comments = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
masks = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentps = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
maskps = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentfs = tf.placeholder(tf.float32, [None, config["numCommentfs"]])
labels = tf.placeholder(tf.float32, [None, config["numClasses"]])
dropoutKeepProb = tf.placeholder(tf.float32)
learningRate = tf.placeholder(tf.float32)
# Create embedding tranform.
with tf.variable_scope("embedding"):
E = tf.get_variable("E", initializer=embed, trainable=False)
embeddings = tf.nn.embedding_lookup(E, comments)
embeddingps = tf.nn.embedding_lookup(E, commentps)
# LSTM
lstmOutputs = None
if config["attentionUnits"]:
lstmOutputs = [
getAttentionLSTMOutputs(embeddings, masks, dropoutKeepProb, "lstm",
config)
]
else:
lstmOutputs = [
getLSTMOutputs(embeddings, masks, dropoutKeepProb, "lstm", config)
]
if config["addCommentp"]:
lstmOutputs.append(
getLSTMOutputs(embeddingps, maskps, dropoutKeepProb, "lstmp",
config))
if config["addCommentf"]:
lstmOutputs.append(commentfs)
lstmOutputs = tf.concat(lstmOutputs, axis=1)
# Layer 1 ReLu
W1 = tf.get_variable(
"W1",
shape=[config["numLSTMOutputs"], config["layer2Units"]],
initializer=tf.initializers.truncated_normal())
b1 = tf.get_variable("b1",
shape=[config["layer2Units"]],
initializer=tf.constant_initializer(0.1))
layer1Output = tf.nn.relu(tf.matmul(lstmOutputs, W1) + b1)
# Dropout
layer1Droutput = tf.nn.dropout(layer1Output, dropoutKeepProb)
# layer 2 softmax
with tf.variable_scope("layer2"):
W2 = tf.get_variable(
"W2",
shape=[config["layer2Units"], config["numClasses"]],
initializer=tf.initializers.truncated_normal())
b2 = tf.get_variable("b2",
shape=[config["numClasses"]],
initializer=tf.constant_initializer(0.1))
prediction = tf.matmul(layer1Droutput, W2) + b2
# Accuracy
prediction2 = tf.argmax(prediction, 1)
labels2 = tf.argmax(labels, 1)
confusionMatrix = tf.confusion_matrix(labels2, prediction2)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(prediction2, labels2), tf.float32))
# Loss and optimizer
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=prediction,
labels=labels))
optimizer = tf.train.AdamOptimizer(
learning_rate=learningRate).minimize(loss)
# Saver
saver = tf.train.Saver()
# Collect Info.
losses = []
trainAccuracies = []
devAccuracies = []
# Collect best info.
bestDevIndex = None
bestDevPredictions = []
bestDevConfusionMatrix = []
with tf.Session() as sess:
# Variable Initialization.
if config["restoreDir"]:
saver.restore(sess, config["restoreDir"])
else:
sess.run(tf.global_variables_initializer())
for epoch in range(config["numEpochs"]):
start = time.time()
# Training.
epochLoss = 0
epochAccuracy = 0
for batchNum, batches in enumerate(
utils.get_minibatches(trainData, config["batchSize"])):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
learningRate: config["learningRates"][epoch],
dropoutKeepProb: config["dropoutKeepProb"]
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchSize = len(batches[0])
batchAccuracy, batchLoss, _ = sess.run(
[accuracy, loss, optimizer], feedDict)
epochLoss += batchLoss * batchSize
epochAccuracy += batchAccuracy * batchSize
if (batchNum + 1) % 100 == 0:
print "Epoch: {}, Batch: {}".format(
epoch + 1, batchNum + 1)
losses.append(epochLoss / float(config["numTrain"]))
trainAccuracies.append(epochAccuracy / float(config["numTrain"]))
print "Epoch: {}, Loss: {}, Accuracy: {}".format(
epoch + 1, losses[-1], trainAccuracies[-1])
# Dev.
epochPredictions = []
epochConfusionMatrix = 0
epochAccuracy = 0
for batchNum, batches in enumerate(
utils.get_minibatches(devData, config["batchSize"],
False)):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
learningRate: config["learningRates"][epoch],
dropoutKeepProb: 1.0
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchPredictions, batchConfusionMatrix, batchAccuracy = sess.run(
[prediction2, confusionMatrix, accuracy], feedDict)
batchSize = len(batches[0])
epochPredictions.extend(batchPredictions)
epochConfusionMatrix += np.asarray(batchConfusionMatrix)
epochAccuracy += batchAccuracy * batchSize
devAccuracies.append(epochAccuracy / float(config["numDev"]))
print "Dev Accuracy: {}".format(devAccuracies[-1])
print epochConfusionMatrix
# Save best dev
if bestDevIndex is None or devAccuracies[-1] > devAccuracies[
bestDevIndex]:
bestDevIndex = len(devAccuracies) - 1
bestDevPredictions = epochPredictions
bestDevConfusionMatrix = epochConfusionMatrix
if config["saveDir"]:
savePath = saver.save(sess, config["saveDir"])
print "Variables saved at {}".format(savePath)
print "Epoch Time: {0:.4f} s".format(time.time() - start)
print ""
# Print out summary.
print "Best Dev of {} at epoch {}, train acc: {}, train loss: {}".format(
devAccuracies[bestDevIndex], bestDevIndex + 1,
trainAccuracies[bestDevIndex], losses[bestDevIndex])
print bestDevConfusionMatrix
# Return series.
return losses, trainAccuracies, devAccuracies, bestDevPredictions
def getAlphas(embed, devData, config):
# Create input placeholders
comments = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
masks = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentps = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
maskps = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentfs = tf.placeholder(tf.float32, [None, config["numCommentfs"]])
labels = tf.placeholder(tf.float32, [None, config["numClasses"]])
dropoutKeepProb = tf.placeholder(tf.float32)
# Create embedding tranform.
with tf.variable_scope("embedding"):
E = tf.get_variable("E", initializer=embed, trainable=False)
embeddings = tf.nn.embedding_lookup(E, comments)
embeddingps = tf.nn.embedding_lookup(E, commentps)
# LSTM
lstmOutputs, alphas = getAttentionLSTMOutputs(embeddings, masks, dropoutKeepProb, "lstm", config)
lstmOutputs = [lstmOutputs]
if config["addCommentp"]:
lstmOutputs.append(getLSTMOutputs(embeddingps, maskps, dropoutKeepProb, "lstmp", config))
if config["addCommentf"]:
lstmOutputs.append(commentfs)
lstmOutputs = tf.concat(lstmOutputs, axis=1)
# Layer 1 ReLu
W1 = tf.get_variable(
"W1",
shape=[config["numLSTMOutputs"], config["layer2Units"]],
initializer=tf.initializers.truncated_normal())
b1 = tf.get_variable(
"b1",
shape=[config["layer2Units"]],
initializer=tf.constant_initializer(0.1))
layer1Output = tf.nn.relu(tf.matmul(lstmOutputs, W1) + b1)
# Dropout
layer1Droutput = tf.nn.dropout(layer1Output, dropoutKeepProb)
# layer 2 softmax
with tf.variable_scope("layer2"):
W2 = tf.get_variable(
"W2",
shape=[config["layer2Units"], config["numClasses"]],
initializer=tf.initializers.truncated_normal())
b2 = tf.get_variable(
"b2",
shape=[config["numClasses"]],
initializer=tf.constant_initializer(0.1))
prediction = tf.matmul(layer1Droutput, W2) + b2
# Accuracy
prediction2 = tf.argmax(prediction, 1)
# Saver
saver = tf.train.Saver()
with tf.Session() as sess:
# Variable Initialization.
saver.restore(sess, config["restoreDir"])
# Evaluate on dev and get attention weights.
predictions = []
allAlphas = None
for batchNum, batches in enumerate(utils.get_minibatches(devData, config["batchSize"], False)):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
dropoutKeepProb: 1.0
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchAlphas, = sess.run([alphas], feedDict)
if allAlphas is None:
allAlphas = batchAlphas
else:
allAlphas = np.concatenate((allAlphas, batchAlphas), axis=0)
if (batchNum + 1) % 100 == 0:
print "Batch: {}".format(batchNum + 1)
# Return series.
return allAlphas
def nerf_iteration(nerf, cfg, rays, near_val, far_val, mode='train'):
batches = get_minibatches(rays, getattr(cfg, mode).chunksize)
coarse, fine = [], []
for batch in batches:
ray_ori, ray_dir = batch[..., :3], batch[..., 3:]
# TODO NDC option
near = near_val * torch.ones_like(ray_ori[..., :1])
far = far_val * torch.ones_like(ray_ori[..., :1])
t = torch.linspace(0., 1., getattr(cfg, mode).num_coarse).to(ray_ori)
# TODO: lindisp option
z_vals = near * (1. - t) + far * t
# basically eq. 2 in the paper
if getattr(cfg, mode).perturb:
mids = .5 * (z_vals[..., 1:] + z_vals[..., :-1])
upper = torch.cat((mids, z_vals[..., -1:]), dim=-1)
lower = torch.cat((z_vals[..., :1], mids), dim=-1)
rand = torch.rand(z_vals.shape).to(z_vals)
z_vals = lower + (upper - lower) * rand
x_xyz = ray_ori[...,
None, :] + ray_dir[..., None, :] * z_vals[..., :, None]
out = nerf.predict_coarse(x_xyz, ray_dir, getattr(cfg, mode).chunksize)
rgb_coarse, weights = volume_render_radiance_field(
out, z_vals, ray_dir,
getattr(cfg, mode).radiance_noise_std,
getattr(cfg, mode).white_background)
coarse.append(rgb_coarse)
if nerf.model_fine is not None:
z_vals_mid = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
z_samples = sample_pdf(
z_vals_mid,
weights[..., 1:-1],
getattr(cfg, mode).num_fine,
det=(getattr(cfg, mode).perturb == 0.0),
)
# important: backprop fine loss online to fine network
# otherwise the sampling is backpropagated to the coarse network
z_samples = z_samples.detach()
z_vals, _ = torch.sort(torch.cat((z_vals, z_samples), dim=-1),
dim=-1)
x_xyz = ray_ori[
..., None, :] + ray_dir[..., None, :] * z_vals[..., :, None]
out = nerf.predict_fine(x_xyz, ray_dir,
getattr(cfg, mode).chunksize)
rgb_fine, _ = volume_render_radiance_field(
out, z_vals, ray_dir,
getattr(cfg, mode).radiance_noise_std,
getattr(cfg, mode).white_background)
fine.append(rgb_fine)
if nerf.model_fine is None:
return torch.cat(coarse, dim=0), None
return torch.cat(coarse, dim=0), torch.cat(fine, dim=0)
def train(embed, trainData, devData, config):
# Create input placeholders
comments = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
masks = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentps = tf.placeholder(tf.int32, [None, config["maxDocLength"]])
maskps = tf.placeholder(tf.float32, [None, config["maxDocLength"]])
commentfs = tf.placeholder(tf.float32, [None, config["numCommentfs"]])
labels = tf.placeholder(tf.float32, [None, config["numClasses"]])
dropoutKeepProb = tf.placeholder(tf.float32)
learningRate = tf.placeholder(tf.float32)
# Create embedding tranform.
with tf.variable_scope("embedding"):
E = tf.get_variable("E", initializer=embed, trainable=False)
embeddings = tf.expand_dims(tf.nn.embedding_lookup(E, comments), -1)
embeddingps = tf.expand_dims(tf.nn.embedding_lookup(E, commentps), -1)
# CNN
cnnOutputs = getCNNOutputs(embeddings, dropoutKeepProb, "c", config)
numFiltersTotal = config["numFilters"] * len(config["filterSizes"])
if config["addCommentp"]:
cnnOutputps = getCNNOutputs(embeddingps, dropoutKeepProb, "p", config)
S = tf.get_variable(
"S",
shape=[numFiltersTotal, numFiltersTotal],
initializer=tf.initializers.truncated_normal(stddev=0.1))
cnnOutputps = tf.expand_dims(
tf.reduce_sum(tf.matmul(cnnOutputs, S) * cnnOutputps, axis=1), -1)
cnnOutputs = tf.concat([cnnOutputs, cnnOutputps], axis=1)
numFiltersTotal += 1
# Dropout
hDroutput = tf.nn.dropout(cnnOutputs, dropoutKeepProb)
# Add in extra features
if config["addCommentf"]:
hDroutput = tf.concat([hDroutput, commentfs], axis=1)
# Layer 1 ReLu
with tf.variable_scope("layer1"):
W1 = tf.get_variable(
"W1",
shape=[
numFiltersTotal + config["numCommentfs"], config["layer2Units"]
],
initializer=tf.initializers.truncated_normal(stddev=0.1))
b1 = tf.get_variable("bias1",
shape=[config["layer2Units"]],
initializer=tf.constant_initializer(0.1))
layer1Output = tf.nn.relu(tf.matmul(hDroutput, W1) + b1)
# Dropout
layer1Droutput = tf.nn.dropout(layer1Output, dropoutKeepProb)
# Output
with tf.variable_scope("layer2"):
W2 = tf.get_variable(
"W2",
shape=[config["layer2Units"], config["numClasses"]],
initializer=tf.initializers.truncated_normal(stddev=0.1))
b2 = tf.get_variable("bias2",
shape=[config["numClasses"]],
initializer=tf.constant_initializer(0.1))
prediction = tf.matmul(layer1Droutput, W2) + b2
# Accuracy
prediction2 = tf.argmax(prediction, 1)
labels2 = tf.argmax(labels, 1)
confusionMatrix = tf.confusion_matrix(labels2, prediction2)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(prediction2, labels2), tf.float32))
# Loss and optimizer
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=prediction,
labels=labels))
if config["lambda"]:
l2 = config["lambda"] * sum(
tf.nn.l2_loss(variable) for variable in tf.trainable_variables()
if not ("bias" in variable.name))
loss += l2
optimizer = tf.train.AdamOptimizer(
learning_rate=learningRate).minimize(loss)
# Saver
saver = tf.train.Saver()
# Collect Info.
losses = []
trainAccuracies = []
devAccuracies = []
# Collect best info.
bestDevIndex = None
bestDevPredictions = []
bestDevConfusionMatrix = []
with tf.Session() as sess:
# Variable Initialization.
if config["restoreDir"]:
print "Restoring from {}".format(config["restoreDir"])
saver.restore(sess, config["restoreDir"])
else:
sess.run(tf.global_variables_initializer())
for epoch in range(config["numEpochs"]):
start = time.time()
# Training.
epochLoss = 0
epochAccuracy = 0
for batchNum, batches in enumerate(
utils.get_minibatches(trainData, config["batchSize"])):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
learningRate: config["learningRates"][epoch],
dropoutKeepProb: config["dropoutKeepProb"]
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchSize = len(batches[0])
batchAccuracy, batchLoss, _ = sess.run(
[accuracy, loss, optimizer], feedDict)
epochLoss += batchLoss * batchSize
epochAccuracy += batchAccuracy * batchSize
if (batchNum + 1) % 100 == 0:
print "Epoch: {}, Batch: {}".format(
epoch + 1, batchNum + 1)
losses.append(epochLoss / float(config["numTrain"]))
trainAccuracies.append(epochAccuracy / float(config["numTrain"]))
print "Epoch: {}, Loss: {}, Accuracy: {}".format(
epoch + 1, losses[-1], trainAccuracies[-1])
# Dev.
epochPredictions = []
epochConfusionMatrix = 0
epochAccuracy = 0
for batchNum, batches in enumerate(
utils.get_minibatches(devData, config["batchSize"],
False)):
feedDict = {
comments: batches[0],
masks: batches[1],
labels: batches[5],
learningRate: config["learningRates"][epoch],
dropoutKeepProb: 1.0
}
if config["addCommentp"]:
feedDict[commentps] = batches[2]
feedDict[maskps] = batches[3]
if config["addCommentf"]:
feedDict[commentfs] = batches[4]
batchPredictions, batchConfusionMatrix, batchAccuracy = sess.run(
[prediction2, confusionMatrix, accuracy], feedDict)
batchSize = len(batches[0])
epochPredictions.extend(batchPredictions)
epochConfusionMatrix += np.asarray(batchConfusionMatrix)
epochAccuracy += batchAccuracy * batchSize
devAccuracies.append(epochAccuracy / float(config["numDev"]))
precision = epochConfusionMatrix[1][1] / float(
epochConfusionMatrix[0][1] + epochConfusionMatrix[1][1])
recall = epochConfusionMatrix[1][1] / float(
epochConfusionMatrix[1][0] + epochConfusionMatrix[1][1])
print "Dev Accuracy: {0:.4f} Precision: {1:.4f} Recall: {2:.4f} F1: {3:.4f}".format(
devAccuracies[-1], precision, recall,
2 * precision * recall / (precision + recall))
print epochConfusionMatrix
# Save best dev
if bestDevIndex is None or devAccuracies[-1] > devAccuracies[
bestDevIndex]:
bestDevIndex = len(devAccuracies) - 1
bestDevPredictions = epochPredictions
bestDevConfusionMatrix = epochConfusionMatrix
if config["saveDir"]:
savePath = saver.save(sess, config["saveDir"])
print "Variables saved at {}".format(savePath)
print "Epoch Time: {0:.4f} s".format(time.time() - start)
print ""
# Print out summary.
print "Best Dev of {} at epoch {}, train acc: {}, train loss: {}".format(
devAccuracies[bestDevIndex], bestDevIndex + 1,
trainAccuracies[bestDevIndex], losses[bestDevIndex])
print bestDevConfusionMatrix
# Return series.
return losses, trainAccuracies, devAccuracies, bestDevPredictions
#
# os.makedirs("saved_models", exist_ok=True)
# torch.save(model.state_dict(), "saved_models/transE.pth")
model.load_state_dict(torch.load("saved_models/transE.pth"))
# n_e=38692
# model = TransE(n_e=40000, n_r=12, k=k, margin=margin, distance=distance, gpu= False)
# model = TransE(n_e=n_e, n_r=n_r, k=k, margin=margin, distance=distance, gpu= False)
X_test = np.load('/data/fazele/workplace/data/wordnet/bin/test.npy')
print(len(np.unique(np.unique(X_test[:, 2]))) + len(np.unique(np.unique(X_test[:, 0]))))
print(len(np.unique(X_test[:, 2])) + len(np.unique(X_test[:, 0])))
print(X_test.min(), X_test.max())
print(len(np.unique(X_test[:,1]) ))
mb_iter = get_minibatches(X_test, 400, shuffle=False)
# for X_mb in mb_iter:
mb_test = next(iter(mb_iter))
# y = model.forward(mb_test)
# print(X_mb.shape)
# y = model.forward(mb_test)
y = model.forward(X_test)
hit_rate = model.hit_at_ten(X_test, 40000)
print("Hit@10 is:", hit_rate)
# model_t = TransE(n_e=n_e, n_r=n_r, k=k, margin=margin, distance=distance, gpu= False)
# y = model(X_test[:600, :])
# pred = model.predict(X_test, sigmoid=True)
