def predict(self, sess, data_set, save_preds=False, UseShuffle=True):
""" Compute predictions on a given data set.
data_set = [headlines, articles, labels]
Return predictions and score
"""
# Compute Predictions
if data_set[0].size == 0:
return 0, []
prog = Progbar(target=1 + len(data_set[0]) / self.config.batch_size)
actual = []
preds = []
for i, (headlines_batch, articles_batch, h_seq_lengths, a_seq_lengths,
labels_batch) in enumerate(
minibatches(data_set, self.config.batch_size, UseShuffle)):
predictions_batch = list(
self.predict_on_batch(sess, headlines_batch, articles_batch,
h_seq_lengths, a_seq_lengths))
preds.extend(predictions_batch)
actual.extend(vectorize_stances(labels_batch))
prog.update(i + 1)
if save_preds:
print "Writing predictions to {}".format(self.preds_fn)
with open(self.preds_fn, 'w') as f:
cPickle.dump(preds, f, -1)
# Compute Score
score, confusion_matrix_str = self.scoring_function(actual, preds)
return score, preds, confusion_matrix_str
def run_epoch(self, sess, train_examples, dev_set):
prog = Progbar(target=1 + train_examples[0].shape[0] / self.config.batch_size)
for i, (articles_batch, headlines_batch, labels_batch) in enumerate(minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess, articles_batch, headlines_batch, labels_batch)
prog.update(i + 1, [("train loss", loss)])
print "Evaluating on train set"
train_actual = vectorize_stances(train_examples[2])
train_preds = list(self.predict_on_batch(sess, *train_examples[:2]))
train_score, _ = report_score(train_actual, train_preds)
print "Evaluating on dev set"
actual = vectorize_stances(dev_set[2])
preds = list(self.predict_on_batch(sess, *dev_set[:2]))
dev_score,_ = report_score(actual, preds)
print "- train Score {:.2f}".format(train_score)
print "- dev Score: {:.2f}".format(dev_score)
return dev_score
def main(debug=True):
if not os.path.exists('./data/weights/'):
os.makedirs('./data/weights/')
with tf.Graph().as_default():
print 80 * "="
print "INITIALIZING"
print 80 * "="
config = Config()
# Load Data
# Note: X_train_input, X_dev_input, X_test_input are tuples consisting of 2 matrices
# the first is the matrix of article representations. The second is the matrix of
# body representations.
X_train_input, X_dev_input, X_test_input, y_train_input, y_dev_input, y_test_input = read_glove_sum_binaries()
# Class weights
class_count = np.sum(y_train_input, axis = 0)
config.class_weights = class_count / np.sum(class_count)
model = SNLI_Baseline_NN(config)
# Create Data Lists
train_examples = [x for x in X_train_input] + [y_train_input]
dev_set = [x for x in X_dev_input] + [y_dev_input]
test_set = [x for x in X_test_input] + [y_test_input]
print "Building model...",
start = time.time()
print "took {:.2f} seconds\n".format(time.time() - start)
init = tf.global_variables_initializer()
saver = None if debug else tf.train.Saver()
with tf.Session() as session:
session.run(init)
print 80 * "="
print "TRAINING"
print 80 * "="
model.fit(session, saver, train_examples, dev_set)
if not debug:
print 80 * "="
print "TESTING"
print 80 * "="
print "Restoring the best model weights found on the dev set"
saver.restore(session, './data/weights/stance.weights')
print "Final evaluation on test set",
actual = vectorize_stances(test_set[2])
preds = list(model.predict_on_batch(session, *test_set[:2]))
_, test_score = report_score(actual, preds)
print "- test Score: {:.2f}".format(test_score)
print "Writing predictions"
with open('snli_nn_baseline_test_predicted.pkl', 'w') as f:
cPickle.dump(preds, f, -1)
print "Done!"
def main():
debug = False
parser = argparse.ArgumentParser()
parser.add_argument('--nn_weights', type=str)
parser.add_argument('--output_file', type=str)
parser.add_argument('--bimpmp', action='store_true')
args = parser.parse_args()
weightFn = args.nn_weights
output_fn = args.output_file
assert weightFn
isBimpmp = args.bimpmp
if isBimpmp:
print "using bimpmp"
config = BimpmpConfig()
else:
print "using bidirattn"
config = BiDirAttnBidirCondConfig()
# Create result directories
linear_related_preds = classify_related_unrelated()
glove_matrix, related_h_glove_index_matrix, related_a_glove_index_matrix, related_h_seq_lengths, related_a_seq_lengths, max_input_lengths, related_labels, unrelated_labels = create_sub_class_test_data(
linear_related_preds, config)
test_set = [
related_h_glove_index_matrix, related_a_glove_index_matrix,
related_h_seq_lengths, related_a_seq_lengths, related_labels
]
sub1_labels = vectorize_stances(unrelated_labels)
with tf.Graph().as_default():
print 80 * "="
print "INITIALIZING"
print 80 * "="
print "Building model...",
scoring_fn = lambda actual, preds: report_pipeline_score(
actual, preds, sub1_labels)
if isBimpmp:
model = Bimpmp(config, scoring_fn, max_input_lengths, glove_matrix,
debug)
else:
model = Bidirectional_Attention_Conditonal_Encoding_LSTM_Model(
config, scoring_fn, max_input_lengths, glove_matrix, debug)
model.print_params()
# Initialize variables
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
saver = create_tensorflow_saver(model.exclude_names)
saver.restore(session, weightFn)
# Finalize graph
session.graph.finalize()
print "TESTING"
test_score, preds, test_confusion_matrix_str = model.predict(
session, test_set, save_preds=True, UseShuffle=False)
print preds
print test_confusion_matrix_str
with open(output_file, 'w') as file:
file.write(test_confusion_matrix_str)
file.write(preds)
def run_epoch(self, sess, train_examples, dev_set):
prog = Progbar(target=1 +
len(train_examples[0]) / self.config.batch_size)
for i, (inputs_batch, labels_batch) in enumerate(
minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess, inputs_batch, labels_batch)
prog.update(i + 1, [("train loss", loss)])
print "Evaluating on dev set"
prog = Progbar(target=1 + len(dev_set[0]) / self.config.batch_size)
actual = []
preds = []
for i, (inputs_batch, labels_batch) in enumerate(
minibatches(dev_set, self.config.batch_size)):
predictions_batch = list(self.predict_on_batch(sess, inputs_batch))
preds.extend(predictions_batch)
actual.extend(vectorize_stances(labels_batch))
prog.update(i + 1)
dev_score, lines = report_score(actual, preds)
print "- dev Score: {:.2f}".format(dev_score)
return dev_score
def main(debug=True):
parser = argparse.ArgumentParser()
parser.add_argument('--epoch', type=int, default=5)
parser.add_argument('--restore', action='store_true')
args = parser.parse_args()
if not os.path.exists('./data/weights/'):
os.makedirs('./data/weights/')
if not os.path.exists('./data/predictions/'):
os.makedirs('./data/predictions/')
if not os.path.exists('./data/plots/'):
os.makedirs('./data/plots/')
with tf.Graph().as_default():
print 80 * "="
print "INITIALIZING"
print 80 * "="
config = Config()
if args.epoch:
config.n_epochs = args.epoch
# Load Data
# Note: X_train_input, X_dev_input, X_test_input are lists where each item is an example.
# Each example is a sparse representation of a headline + article, where the text
# is encoded as a series of indices into the glove-vectors.
# y_train_input, y_dev_input, y_test_input are matrices (num_examples, num_classes)
X_train_input, X_dev_input, X_test_input, y_train_input, y_dev_input, y_test_input, glove_matrix, max_lengths = create_inputs_by_glove(
)
config.max_length = max_lengths[0] + max_lengths[1]
print "Max Length is {}".format(config.max_length)
# Create Basic LSTM Model
config.pretrained_embeddings = glove_matrix
model = BasicLSTM(config)
# Create Data Lists
train_examples = [X_train_input, y_train_input]
dev_set = [X_dev_input, y_dev_input]
test_set = [X_test_input, y_test_input]
print "Building model...",
start = time.time()
print "took {:.2f} seconds\n".format(time.time() - start)
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
exclude_names = set([
"embedding_matrix:0", "embedding_matrix/Adam:0",
"embedding_matrix/Adam_1:0"
])
saver = create_tensorflow_saver(exclude_names)
if args.restore:
saver.restore(session,
'./data/weights/basic_lstm_curr_stance.weights')
print "Restored weights from ./data/weights/basic_lstm_curr_stance.weights"
print "-------------------------------------------"
session.graph.finalize()
print 80 * "="
print "TRAINING"
print 80 * "="
model.fit(session, saver, train_examples, dev_set)
if saver:
print 80 * "="
print "TESTING"
print 80 * "="
print "Restoring the best model weights found on the dev set"
saver.restore(session,
'./data/weights/basic_lstm_best_stance.weights')
print "Final evaluation on test set",
prog = Progbar(target=1 + len(test_set[0]) / config.batch_size)
actual = vectorize_stances(test_set[1])
preds = []
for i, (inputs_batch, labels_batch) in enumerate(
minibatches(test_set, config.batch_size)):
predictions_batch = list(
model.predict_on_batch(session, inputs_batch))
preds.extend(predictions_batch)
prog.update(i + 1)
test_score, test_lines = report_score(actual, preds)
print "- test Score: {:.2f}".format(test_score)
print "Writing predictions"
with open('./data/predictions/basic_lstm_predicted.pkl',
'w') as f:
cPickle.dump(preds, f, -1)
print "Done!"
def main(debug=True):
# Generate Sample Dataset in 2-dimensional space
# Class 1 is a Gaussian Centered Around (0, -5)
# Class 2 is a Gaussian Centered Around (0, 5)
centroid_1 = np.array([0, -5])
centroid_2 = np.array([0, 5])
cov = np.array([
[1, 0],
[0, 1]
])
size = 500
x1, y1 = np.random.multivariate_normal(centroid_1, cov, size).T
x2, y2 = np.random.multivariate_normal(centroid_2, cov, size).T
labels_1 = np.concatenate([np.array([[1, 0]]) for _ in range(size)], axis=0)
labels_2 = np.concatenate([np.array([[0, 1]]) for _ in range(size)], axis=0)
x = np.concatenate([x1, x2], axis = 0).reshape((-1, 1))
y = np.concatenate([y1, y2], axis = 0).reshape((-1, 1))
# Plot data
plt.plot(x1, y1, 'x')
plt.plot(x2, y2, 'x')
plt.axis('equal')
plt.savefig('plot.png', bbox_inches='tight')
all_data = np.concatenate([x, y], axis=1)
all_labels = np.concatenate([labels_1, labels_2], axis=0)
# Split example data into train and test sets
train_data, test_data, train_labels, test_labels = train_test_split(
all_data, all_labels, test_size=0.2, random_state=42
)
with tf.Graph().as_default():
print 80 * "="
print "INITIALIZING"
print 80 * "="
config = Config()
model = Test_Neural_Network(config)
print "Building model...",
start = time.time()
print "took {:.2f} seconds\n".format(time.time() - start)
init = tf.global_variables_initializer()
saver = None if debug else tf.train.Saver()
with tf.Session() as session:
session.run(init)
print 80 * "="
print "TRAINING"
print 80 * "="
model.fit(session, saver, [train_data, train_labels])
preds = model.predict_on_batch(session, test_data)
print "-----"
print preds
print "-----"
stances = np.array(vectorize_stances(test_labels))
print stances
print "-----"
print "dif"
print sum(preds - stances)