def __init__(self, sequence_length, vocab_size, embedding_size,
filter_sizes, num_filters, margin):
with tf.variable_scope("branches") as double_scope:
self.sim_branch = Siamese(sequence_length, vocab_size,
embedding_size, filter_sizes,
num_filters, margin)
double_scope.reuse_variables()
self.disim_branch = Siamese(sequence_length, vocab_size,
embedding_size, filter_sizes,
num_filters, margin)
# TODO: Modify this to minimize the AUR
self.loss = tf.reduce_mean(self.sim_branch.loss +
self.disim_branch.loss,
name="loss_branches")
def test():
images = np.load('data/images_evaluation.npy')[:20] / 255
images = np.expand_dims(images, axis=-1)
ground = images[:, 0]
model = Siamese()
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
ground_scores = sess.run(model.o1, feed_dict={
model.x1: ground,
})
preds = [[] for _ in images]
for i in trange(len(images)):
batch_scores = sess.run(model.o1, feed_dict={
model.x1: images[i]
})
for score in batch_scores:
dist = np.sum(np.abs(ground_scores - score), axis=-1)
pred = np.argmin(dist)
preds[i].append(pred)
y_true = np.array([[i for _ in images[i]] for i in range(len(images))]).flatten()
y_preds = np.array(preds).flatten()
cm = confusion_matrix(y_true, y_preds)
tp = np.eye(len(cm)) * cm
print(np.sum(tp) / np.sum(cm))
plot_confusion_matrix(cm, np.arange(len(images)))
def test():
test_data = 'data/labels'
files = [join(test_data, f'{i}.png') for i in range(10)]
ground = [cv2.imread(f, 0) / 255 for f in files]
ground = np.array(ground).reshape([-1, 28, 28, 1])
siamese = Siamese()
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
ground_scores = sess.run(siamese.o1, feed_dict={
siamese.x1: ground,
siamese.keep_prob: 1.0,
})
x, y = mnist.test.images, mnist.test.labels
x = np.reshape(x, [-1, h, w, c])
n_correct = 0
batch_size = 1000
n_batches = len(x) // batch_size
for i in trange(n_batches):
batch_images, batch_labels = x[i::n_batches], y[i::n_batches]
batch_scores = sess.run(siamese.o1, feed_dict={
siamese.x1: batch_images,
siamese.keep_prob: 1.0,
})
for score, label in zip(batch_scores, batch_labels):
dist = np.sum(np.abs(ground_scores - score), axis=-1)
pred = np.argmin(dist)
n_correct += (pred == label)
print(n_correct / len(x))
def __init__(self):
self.super_resolution = SuperResolution(
model_path=os.getenv("SUPER_RESOLUTION_DIR"))
self.detector = Detector(os.getenv("DETECTOR_DIR"))
self.extractor = FacenetExtractor(
model_path=os.getenv("EXTRACTOR_DIR"))
self.extractor.init_model()
self.utils = Utils(super_resolution=self.super_resolution,
extractor=self.extractor)
self.classifier = Siamese()
self.classifier.load_model(os.getenv("CLASSIFIER_DIR"))
self.db = DbConnection()
def __init__(self, dataset_path="datasets/lfw-dataset", img_size=(36, 36)):
self.super_resolution = SuperResolution(
model_path=os.getenv("SUPER_RESOLUTION_DIR"))
self.detector = Detector(os.getenv("DETECTOR_DIR"))
self.face_extractor = FacenetExtractor(
model_path=os.getenv("EXTRACTOR_DIR"))
self.utils = Utils(super_resolution=self.super_resolution)
self.dataset_path = dataset_path
self.input_size = (160, 160)
self.img_size = img_size
self.face_extractor.init_model()
self.classifier = Siamese()
self.classifier.load_model(os.getenv("CLASSIFIER_DIR"))
def feature_extraction_siamese():
"""
This method restores a TensorFlow checkpoint file (.ckpt) and rebuilds inference
model with restored parameters. From then on you can basically use that model in
any way you want, for instance, feature extraction, finetuning or as a submodule
of a larger architecture. However, this method should extract features from a
specified layer and store them in data files such as '.h5', '.npy'/'.npz'
depending on your preference. You will use those files later in the assignment.
Args:
[optional]
Returns:
None
"""
########################
# PUT YOUR CODE HERE #
########################
tf.reset_default_graph()
classes = [
'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship',
'truck'
]
tf.set_random_seed(42)
np.random.seed(42)
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
x_test, y_test = cifar10.test.images, cifar10.test.labels
y_test = np.argmax(y_test, axis=1)
input_data_dim = cifar10.test.images.shape[1]
n_classes = 10
cnn_siamese = Siamese()
x = tf.placeholder(tf.float32,
shape=(None, input_data_dim, input_data_dim, 3),
name="x1")
y = tf.placeholder(tf.float32, shape=(None, 1), name="y")
with tf.name_scope('train_cnn'):
infs1 = cnn_siamese.inference(x, reuse=None)
l2_out = cnn_siamese.l2_out
with tf.Session() as sess:
saver = tf.train.Saver()
saver.restore(sess, FLAGS.checkpoint_dir + '/cnn_model_siamese.ckpt')
l2_out_features = sess.run([l2_out], feed_dict={x: x_test})[0]
_plot_tsne("L2 out", l2_out_features, y_test)
_train_one_vs_all(l2_out_features, y_test, "L2 norm", classes)
def main():
siamese = Siamese()
siamese.load_model()
EONS = 10
for k in range(EONS):
print('======= Eon %d/%d ======== ' % (k, EONS))
pairs_train = get_all_pairs()
pairs_test = get_all_pairs()
print('Pairs:', len(pairs_train))
shuffle(pairs_train)
pos = [x for x in pairs_train if x.label == 1]
print('Pos:', len(pos))
neg = [x for x in pairs_train if x.label == 0]
print('Neg:', len(neg))
print((len(pos) + len(neg)))
train_model(siamese, pairs_train, pairs_test)
def main():
siamese = Siamese()
all_pairs = get_all_pairs()
#print('Pairs:', len(all_pairs))
#print(all_pairs[0].print_images())
#for i in range(1, len(all_pairs)):
# index = random.randrange(0, i)
# all_pairs[index], all_pairs[i] = all_pairs[i], all_pairs[index]
#print('Pairs:', len(all_pairs))
#print(all_pairs[0].print_images())
shuffle(all_pairs)
'''for k in range(10):
f, axarr = plt.subplots(1, 2)
axarr[0].imshow(a[k])
axarr[1].imshow(b[k])
if c[k] == 1:
axarr[0].set_title('Same')
axarr[1].set_title('Same')
else:
axarr[0].set_title('Different')
axarr[1].set_title('Different')
# print(a[0] + ' | ' + b[0])
plt.show()'''
pos = [x for x in all_pairs if x.label == 1]
print('Pos:', len(pos))
neg = [x for x in all_pairs if x.label == 0]
print('Neg:', len(neg))
print((len(pos)+len(neg)))
# for par in all_pairs[:30]:
# par.print_label()
# par.print_images()
train_model(siamese, all_pairs)
def train():
learning_rate = 1e-4
num_iterations = 20_000
siamese = Siamese()
optimizer = tf.train.AdamOptimizer(learning_rate)
train_step = optimizer.minimize(siamese.loss)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in trange(num_iterations):
x1, y1 = mnist.train.next_batch(128)
x2, y2 = mnist.train.next_batch(128)
x1 = np.reshape(x1, [-1, 28, 28, 1])
x2 = np.reshape(x2, [-1, 28, 28, 1])
y = (y1 == y2).astype(np.float32)
feed_dict = {
siamese.x1: x1,
siamese.x2: x2,
siamese.y_: y,
siamese.keep_prob: 0.5,
}
_, loss_v = sess.run([train_step, siamese.loss], feed_dict=feed_dict)
assert not np.isnan(loss_v), 'Model diverged with loss = NaN'
if i % 100 == 0:
tqdm.write(f'step {i}: loss {loss_v}')
if i % 1000 == 0:
tqdm.write('Model saved:', saver.save(sess, model_path))
print('Finished:', saver.save(sess, model_path))
def train_siamese_fromtf(tf_path,
flags,
num_epochs,
out_dir=None,
one_hot=False,
verbose=False,
init_embeddings=None):
""" Train a Siamese NN using a tfrecords as an input"""
tf.logging.set_verbosity(tf.logging.INFO)
# Create the directory where the training will be saved
if not out_dir:
timestamp = str(int(time()))
out_dir = abspath(join(curdir, "models", timestamp))
makedirs(out_dir, exist_ok=True)
# Load the records
train_path = join(tf_path, 'train.tfrecords')
vocab_processor_path = join(tf_path, 'vocab.train')
vocab_processor = load_binarize_data(vocab_processor_path)
sequence_length_path = join(tf_path, 'sequence.len')
seq_len = load_binarize_data(sequence_length_path)
# Read the configuration flags
# TODO Remove this from the siamese class
n_labels = 2 if one_hot else 1
print('--------', n_labels)
with tf.Graph().as_default():
label_batch, sentences_1_batch, sentences_2_batch = input_pipeline(
filepath=train_path,
batch_size=flags.batch_size,
num_labels=n_labels,
sequence_len=seq_len,
num_epochs=num_epochs)
siamese = Siamese(sequence_length=seq_len,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=flags.embedding_dim,
filter_sizes=list(
map(int, flags.filter_sizes.split(","))),
num_filters=flags.num_filters,
margin=flags.margin)
global_step = tf.Variable(0, trainable=False)
# learning_rate = tf.placeholder(tf.float32, shape=[])
# train_op = tf.train.GradientDescentOptimizer(
# learning_rate=learning_rate).minimize(siamese.loss)
# optimizer = tf.train.AdamOptimizer(0.2)
# grads_and_vars = optimizer.compute_gradients(siamese.loss)
# train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
starter_learning_rate = 0.01
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
1000000,
0.95,
staircase=False)
train_op = tf.train.MomentumOptimizer(learning_rate,
0.5,
use_nesterov=True)
# train_op = tf.train.MomentumOptimizer(0.01, 0.5, use_nesterov=True)
train_op = train_op.minimize(siamese.loss, global_step=global_step)
init_op = tf.global_variables_initializer()
init_again = tf.local_variables_initializer()
saver = tf.train.Saver()
session_conf = tf.ConfigProto(
allow_soft_placement=flags.allow_soft_placement,
log_device_placement=flags.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default() as sess:
if verbose:
tf.summary.histogram('embedding', siamese.W_embedding)
tf.summary.histogram('tensor_left', siamese.left_siamese)
# tf.summary.histogram('tensor_left_z', tf.nn.zero_fraction(siamese.left_siamese))
tf.summary.histogram('tensor_right', siamese.right_siamese)
# tf.summary.histogram('tensor_right_z', tf.nn.zero_fraction(siamese.right_siamese))
tf.summary.histogram('distance', siamese.distance)
tf.summary.scalar('loss', siamese.loss)
tf.summary.scalar('distance', siamese.distance[0])
tf.summary.scalar('attraction', siamese.attr[0][0])
tf.summary.scalar('repulsion', siamese.rep[0][0])
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter('./train', sess.graph)
sess.run(init_op)
sess.run(init_again)
# Show which variables are going to be train
variables_names = [v.name for v in tf.trainable_variables()]
values = sess.run(variables_names)
for k, v in zip(variables_names, values):
print("Variable: ", k, "- Shape: ", v.shape)
# Load embeddings
if init_embeddings is not None:
sess.run(siamese.W_embedding.assign(init_embeddings))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
try:
while not coord.should_stop():
labels, s1, s2 = sess.run(
[label_batch, sentences_1_batch, sentences_2_batch])
current_step = tf.train.global_step(sess, global_step)
if verbose:
train_step_verbose(sess, train_op, summary_op,
summary_writer, siamese, s1, s2,
labels, current_step)
else:
train_step(sess, train_op, siamese, s1, s2, labels,
out_dir, current_step)
except tf.errors.OutOfRangeError:
print("Done training!")
finally:
coord.request_stop()
coord.join(threads)
# Save the model
if not out_dir:
timestamp = str(int(time()))
out_dir = abspath(join(curdir, "models", timestamp))
makedirs(out_dir, exist_ok=True)
with open(join(out_dir, 'parameters.txt'), 'w') as param_file:
param_file.write("Default parameters: \n")
for attr, value in sorted(flags.__flags.items()):
param_file.write(" - {}={}\n".format(attr.upper(), value))
save_path = saver.save(sess, join(out_dir, "model.ckpt"))
print("Model saved in file: {}".format(save_path))
return out_dir
def main():
config_filename = Path.cwd().joinpath(CONFIGS_DIR).joinpath(
CONFIG_FILENAME)
config = Configuration(config_filename)
batch_size = 4
epochs = 4
results_dir_path = Path.cwd().joinpath(RESULTS_DIR)
current_run_path = create_results_directories(results_dir_path)
sample_logger_path = Path.cwd().joinpath(current_run_path).joinpath(
SAMPLE_LOGGER_FILE)
sample_logger = SampleLogger(sample_logger_path)
transforms = TransformsComposer([Rescale(output_size=10000), ToTensor()])
encoder = LabelEncoder()
data_loader = DataLoader(config)
x_train, y_train = data_loader.get_train_set()
encoder.fit(y_train)
classes = encoder.classes_
classes_map = {}
for i, category in enumerate(classes):
classes_map[i] = category
print(classes_map)
y_train = encoder.transform(y_train)
train_dataset = SimilarityDataset(x_train, y_train, classes_map,
sample_logger, transforms)
x_test, y_test = data_loader.get_test_set()
y_test = encoder.transform(y_test)
test_dataset = SimilarityDataset(x_test, y_test, classes_map,
sample_logger, transforms)
model = Siamese()
states_dir = Path.cwd().joinpath(STATES_DIR)
state_filename = f'{uuid.uuid1()}_state_{epochs}_epochs.pth'
state_path = current_run_path.joinpath('best_snapshot').joinpath(
state_filename)
classifier = SimilarityClassifier(model=model, state_path=state_path)
# Fit model on data
train_loss_history, val_loss_history = classifier.fit(
train_dataset,
batch_size=batch_size,
epochs=epochs,
validation_data=test_dataset)
sample_logger.save()
# plt.figure()
# plt.title(f'Model Loss for {epochs} epochs')
# plt.xlabel('epoch')
# plt.ylabel('loss')
# plt.plot(train_loss_history, label='train')
# plt.plot(val_loss_history, label='test')
# plt.legend()
# plt.show()
predictions_path = Path.cwd().joinpath('./predicted.csv')
validation_dataset = SimilarityDataset(x_test, y_test, classes_map,
sample_logger, transforms)
validation_model = Siamese(num_classes=len(classes_map))
validation_classifier = SimilarityClassifier(validation_model,
state_path=state_path)
validation_classifier.predict(validation_dataset,
batch_size=batch_size,
output_filepath=predictions_path)
def feature_extraction():
"""
This method restores a TensorFlow checkpoint file (.ckpt) and rebuilds inference
model with restored parameters. From then on you can basically use that model in
any way you want, for instance, feature extraction, finetuning or as a submodule
of a larger architecture. However, this method should extract features from a
specified layer and store them in data files such as '.h5', '.npy'/'.npz'
depending on your preference. You will use those files later in the assignment.
Args:
[optional]
Returns:
None
"""
########################
# PUT YOUR CODE HERE #
########################
print("extract features")
if FLAGS.train_model == 'linear':
cnet = ConvNet()
cifar10 = cifar10_utils.get_cifar10()
else:
cnet = Siamese()
cifar10 = cifar10_utils.get_cifar10()
x_in = tf.placeholder(tf.float32, [None, 32, 32, 3])
y_true = tf.placeholder(tf.float32, [None, 10])
if FLAGS.train_model == 'siamese':
x_anchor = tf.placeholder(tf.float32, [None, 32, 32, 3])
with tf.variable_scope("Siamese", reuse=None):
filter1 = tf.get_variable("filter1",
initializer=tf.random_normal(
[5, 5, 3, 64], dtype=tf.float32))
filter2 = tf.get_variable("filter2",
initializer=tf.random_normal(
[5, 5, 64, 64], dtype=tf.float32))
W1 = tf.get_variable("W1",
initializer=tf.random_normal(
[4096, 384], dtype=tf.float32))
W2 = tf.get_variable("W2",
initializer=tf.random_normal(
[384, 192], dtype=tf.float32))
else:
with tf.variable_scope("ConvNet", reuse=None):
filter1 = tf.get_variable("filter1",
initializer=tf.random_normal(
[5, 5, 3, 64], dtype=tf.float32))
filter2 = tf.get_variable("filter2",
initializer=tf.random_normal(
[5, 5, 64, 64], dtype=tf.float32))
W1 = tf.get_variable("W1",
initializer=tf.random_normal(
[4096, 384], dtype=tf.float32))
W2 = tf.get_variable("W2",
initializer=tf.random_normal(
[384, 192], dtype=tf.float32))
W3 = tf.get_variable("W3",
initializer=tf.random_normal(
[192, 10], dtype=tf.float32))
loader = tf.train.Saver()
sess = tf.Session()
ts = TSNE(n_components=2, perplexity=1)
if FLAGS.train_model == 'linear':
loader.restore(sess,
FLAGS.checkpoint_dir + "/ConvNet/" + "checkpoint.ckpt")
flatten = np.load(FLAGS.checkpoint_dir + "/ConvNet/flatten.npy")
fc1 = np.load(FLAGS.checkpoint_dir + "/ConvNet/fc1.npy")
fc2 = np.load(FLAGS.checkpoint_dir + "/ConvNet/fc2.npy")
labels = np.load(FLAGS.checkpoint_dir + "/ConvNet/labels.npy")
f2 = ts.fit_transform(fc2)
f1 = ts.fit_transform(fc1)
ff = ts.fit_transform(flatten)
plot1 = plt.scatter(ff[:, 0], ff[:, 1], color=labels.argmax(1))
plt.savefig("convflatten.png")
plot2 = plt.scatter(f1[:, 0], f1[:, 1], color=labels.argmax(1))
plt.savefig("convfc1.png")
plot3 = plt.scatter(f2[:, 0], f2[:, 1], color=labels.argmax(1))
plt.savefig("convfc2.png")
#1vsrest
lvr = OneVsRestClassifier(LinearSVC())
lvrf2 = lvr.fit(f2, labels).predict(f2)
lvrf1 = lvr.fit(f1, labels).predict(f1)
lvrff = lvr.fit(ff, labels).predict(ff)
classes = range(10)
for i in classes:
accf2 = np.mean(1 * (lvrf2 == labels))
accf1 = np.mean(1 * (lvrf1 == labels))
accff = np.mean(1 * (lvrff == labels))
sys.stderr.write("for class " + str(i) +
", OVR accuracies are: \n" + "\t" + str(accf2) +
"\t for fc2\n" + "\t" + str(accf1) +
"\t for fc1\n" + "\t" + str(accf2) +
"\t for flatten\n")
else:
loader.restore(sess,
FLAGS.checkpoint_dir + "/Siamese/" + "checkpoint.ckpt")
lo = np.load(FLAGS.checkpoint_dir + "/Siamese/other.npy")
loa = np.load(FLAGS.checkpoint_dir + "/Siamese/anchor.npy")
lop = ts.fit_transform(lo)
loap = ts.fit_transform(loa)
plot1 = plt.scatter(loap[:, 0], loap[:, 1])
plt.savefig("siamanchor.png")
plot2 = plt.scatter(lop[:, 0], lop[:, 1])
plt.savefig("siamother.png")
def train_siamese():
"""
Performs training and evaluation of Siamese model.
First define your graph using class Siamese and its methods. Then define
necessary operations such as trainer (train_step in this case), savers
and summarizers. Finally, initialize your model within a tf.Session and
do the training.
---------------------------
How to evaluate your model:
---------------------------
On train set, it is fine to monitor loss over minibatches. On the other
hand, in order to evaluate on test set you will need to create a fixed
validation set using the data sampling function you implement for siamese
architecture. What you need to do is to iterate over all minibatches in
the validation set and calculate the average loss over all minibatches.
---------------------------------
How often to evaluate your model:
---------------------------------
- on training set every print_freq iterations
- on test set every eval_freq iterations
------------------------
Additional requirements:
------------------------
Also you are supposed to take snapshots of your model state (i.e. graph,
weights and etc.) every checkpoint_freq iterations. For this, you should
study TensorFlow's tf.train.Saver class. For more information, please
checkout:
[https://www.tensorflow.org/versions/r0.11/how_tos/variables/index.html]
"""
# Set the random seeds for reproducibility. DO NOT CHANGE.
tf.set_random_seed(42)
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
########################
weight_init_scale = 0.001
cifar10 = cifar10_siamese_utils.get_cifar10(validation_size=500)
cnet = Siamese()
#swriter = tf.train.SummaryWriter(FLAGS.log_dir + "/Siamese/")
x_anchor = tf.placeholder(tf.float32, [None, 32, 32, 3])
x_in = tf.placeholder(tf.float32, [None, 32, 32, 3])
y_true = tf.placeholder(tf.float32, [None])
with tf.variable_scope("Siamese", reuse=None):
filter1 = tf.get_variable("filter1",
initializer=tf.random_normal(
[5, 5, 3, 64],
stddev=weight_init_scale,
dtype=tf.float32))
filter2 = tf.get_variable("filter2",
initializer=tf.random_normal(
[5, 5, 64, 64],
stddev=weight_init_scale,
dtype=tf.float32))
W1 = tf.get_variable("W1",
initializer=tf.random_normal(
[4096, 384],
stddev=weight_init_scale,
dtype=tf.float32))
W2 = tf.get_variable("W2",
initializer=tf.random_normal(
[384, 192],
stddev=weight_init_scale,
dtype=tf.float32))
sess = tf.Session()
saver = tf.train.Saver()
#define things
logits_anchor, _f1, _f2, _fl = cnet.inference(x_anchor)
logits_in, _f1, _f2, _fl = cnet.inference(x_in)
loss = cnet.loss(logits_anchor, logits_in, y_true, 1.0)
opt_iter = train_step(loss)
sess.run(tf.initialize_all_variables())
#xbat, ybat = cifar10.train.next_batch(100)
#begin the training
with sess:
# loop
for i in range(FLAGS.max_steps + 1):
ancbat, xbat, ybat = cifar10.train.next_batch(FLAGS.batch_size)
sess.run(opt_iter,
feed_dict={
x_anchor: ancbat,
x_in: xbat,
y_true: ybat
})
if i % FLAGS.print_freq == 0:
ancbat, xbat, ybat = cifar10.validation.next_batch(100)
val_loss = sess.run([loss],
feed_dict={
x_anchor: ancbat,
x_in: xbat,
y_true: ybat
})
sys.stderr.write("iteration : " + str(i) +
", validation loss : " + str(val_loss) + "\n")
#swriter.add_summary(
# sess.run(tf.scalar_summary("loss", val_loss),
# feed_dict = {x_anchor: ancbat, x_in: xbat, y_true:ybat})
# ,i)
if i % FLAGS.checkpoint_freq == 0:
saver.save(
sess,
FLAGS.checkpoint_dir + "/Siamese/" + "checkpoint.ckpt")
lo, flatsave, fc1save, fc2save = sess.run(cnet.inference(x_in),
feed_dict={
x_in: xbat,
y_true: ybat,
x_anchor: ancbat
})
loa, flatsavea, fc1savea, fc2savea = sess.run(
cnet.inference(x_anchor),
feed_dict={
x_in: xbat,
y_true: ybat,
x_anchor: ancbat
})
np.save(FLAGS.checkpoint_dir + "/Siamese/other", lo)
np.save(FLAGS.checkpoint_dir + "/Siamese/anchor", loa)
"""
np.save(FLAGS.checkpoint_dir +"/Siamese/flatten", flatsave)
np.save(FLAGS.checkpoint_dir + "/Siamese/fc1", fc1save)
np.save(FLAGS.checkpoint_dir + "/Siamese/fc2", fc2save)
np.save(FLAGS.checkpoint_dir +"/Siamese/flattena", flatsavea)
np.save(FLAGS.checkpoint_dir + "/Siamese/fc1a", fc1savea)
np.save(FLAGS.checkpoint_dir + "/Siamese/fc2a", fc2savea)
"""
if i % FLAGS.eval_freq == 0:
ancbat, xbat, ybat = cifar10.test.next_batch(100)
sys.stderr.write("test loss:" + str(
sess.run(loss,
feed_dict={
x_anchor: ancbat,
x_in: xbat,
y_true: ybat
})) + "\n")
def train_siamese(train_non_sim,
train_sim,
dev_non_sim,
dev_sim,
vocab_processor,
sequence_len,
config_flags=None):
""" Train a siamese NN """
FLAGS = read_flags(config_flags)
val_left_sentences, val_right_sentences, val_sim_labels = get_dev_data(
dev_sim, dev_non_sim)
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
# TODO Remove this from the siamese class
if not FLAGS.hash_size:
fully_layer = False
else:
fully_layer = True
with sess.as_default():
print('HASH TRAIN ----->', FLAGS.hash_size)
siamese = Siamese(sequence_len,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
margin=FLAGS.margin,
threshold=FLAGS.threshold,
fully=fully_layer,
hash_size=FLAGS.hash_size)
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = 0.1
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
100000,
0.96,
staircase=True)
train_step = tf.train.MomentumOptimizer(
0.0001, 0.95,
use_nesterov=True).minimize(siamese.loss,
global_step=global_step)
print()
sess.run(tf.global_variables_initializer())
data_size = len(train_sim) + len(train_non_sim)
num_batches_per_epoch = int(data_size / FLAGS.batch_size) + 1
print("Num batches per epoch: {} ({})\n".format(
num_batches_per_epoch, data_size))
train_sim = np.array(train_sim)
train_non_sim = np.array(train_non_sim)
for epoch in range(FLAGS.num_epochs):
print(
"-------------------------------- EPOCH {} ---------------------------"
.format(epoch))
# Prepare the batches
if FLAGS.shuffle_epochs:
shuffled_sim_data, shuffled_non_sim_data = shuffle_epochs(
train_sim, train_non_sim)
batches = batch_iter(shuffled_sim_data,
shuffled_non_sim_data,
FLAGS.batch_size,
num_batches_per_epoch)
else:
batches = batch_iter(train_sim, train_non_sim,
FLAGS.batch_size,
num_batches_per_epoch)
# TRAIN A BATCH
sim_distances, non_sim_distances = [], []
for cur_batch, batch in enumerate(batches):
batch_data, batch_type = batch[0], batch[1]
right_sentences = [
sample.sentence_1 for sample in batch_data
]
left_sentences = [
sample.sentence_2 for sample in batch_data
]
sim_labels = [sample.label for sample in batch_data]
# print(Counter(sim_labels))
# print(len(right_sentences))
assert len(right_sentences) == len(left_sentences) == len(
sim_labels)
_, loss, attraction, repulsion, d, accuracy, predictions, correct = sess.run(
[
train_step, siamese.loss, siamese.attraction_loss,
siamese.repulsion_loss, siamese.distance,
siamese.accuracy, siamese.predictions,
siamese.correct_predictions
],
feed_dict={
siamese.left_input: left_sentences,
siamese.right_input: right_sentences,
siamese.label: sim_labels
})
print("(#{0: <7}) - Loss: {1:.4f} (a: {2:.4f} - r: {3:.4f}"
"- d: {4:.4f}, accuracy:{5:.4f})".format(
batch_type, loss, np.mean(attraction),
np.mean(repulsion), np.mean(d), accuracy))
if batch_type == 'SIM':
sim_distances.append(d)
else:
non_sim_distances.append(d)
print('---------------------> sim: {} - non sim: {}'.format(
np.array(sim_distances).mean(),
np.array(non_sim_distances).mean()))
print(len(val_sim_labels))
dev_step(sess, siamese, val_left_sentences,
val_right_sentences, val_sim_labels, epoch)
print('Working dev step')
def train_siamese_fromtf(tf_path, config_flags, one_hot=False):
""" Train a Siamese NN using a tfrecords as an input"""
# Load the records
train_path = join(tf_path, 'train.tfrecords')
vocab_processor_path = join(tf_path, 'vocab.train')
vocab_processor = load_binarize_data(vocab_processor_path)
sequence_length_path = join(tf_path, 'sequence.len')
seq_len = load_binarize_data(sequence_length_path)
# Read the configuration flags
FLAGS = read_flags(config_flags)
# TODO Remove this from the siamese class
fully_layer = True if FLAGS.hash_size else False
n_labels = 2 if one_hot else 1
print('--------', n_labels)
# Load the dev records
dev_path = join(tf_path, 'dev.tfrecords')
# dev_labels, dev_s1, dev_s2 = get_all_records(dev_path, n_labels, seq_len)
with tf.Graph().as_default():
label_batch, sentences_1_batch, sentences_2_batch = input_pipeline(
filepath=train_path,
batch_size=FLAGS.batch_size,
num_labels=n_labels,
sequence_len=seq_len,
num_epochs=FLAGS.num_epochs)
#
# dev_labels, dev_sentences_1, dev_sentences_2 = input_pipeline(filepath=dev_path,
# batch_size=FLAGS.batch_size,
# num_labels=n_labels,
# sequence_len=seq_len,
# num_epochs=FLAGS.num_epochs)
#
print('HASH TRAIN ----->', FLAGS.hash_size)
siamese = Siamese(sequence_length=seq_len,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
margin=FLAGS.margin,
threshold=FLAGS.threshold,
fully=fully_layer,
hash_size=FLAGS.hash_size)
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = 0.1
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
100000,
0.96,
staircase=True)
train_op = tf.train.MomentumOptimizer(
0.0001, 0.95, use_nesterov=True).minimize(siamese.loss,
global_step=global_step)
init_op = tf.global_variables_initializer()
init_again = tf.local_variables_initializer()
saver = tf.train.Saver()
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default() as sess:
sess.run(init_op)
sess.run(init_again)
# TODO la funcion map no la detecta y por lo tanto NO VA NADA
# training_dataset = tf.contrib.data.TFRecordDataset([train_path])
# # training_dataset = training_dataset.map(lambda x: parse_function(x, n_labels, seq_len))
# training_dataset = training_dataset.map(lambda x: x)
#
# # training_dataset = training_dataset.shuffle(buffer_size=10000)
# # training_dataset = training_dataset.repeat().batch(100)
#
# validation_dataset = tf.contrib.data.TFRecordDataset([train_path])
# # training_dataset = tf.contrib.data.TFRecordDataset([train_path]).map(lambda x: )
# # validation_dataset = tf.contrib.data.TFRecordDataset([train_path]).map(
# # lambda x: parse_function(x, n_labels, seq_len))
# iterator = tf.contrib.data.Iterator.from_structure(training_dataset.output_types,
# training_dataset.output_shapes)
# next_element = iterator.get_next()
#
# training_init_op = iterator.make_initializer(training_dataset)
# validation_init_op = iterator.make_initializer(training_dataset)
#
# # Run 20 epochs in which the training dataset is traversed, followed by the
# # validation dataset.
# for _ in range(1):
# # Initialize an iterator over the training dataset.
# sess.run(training_init_op)
# for _ in range(1):
# a = sess.run(next_element)
# # parse_function(a, n_labels, seq_len)
# print(a)
# #
# # # Initialize an iterator over the validation dataset.
# # sess.run(validation_init_op)
# # for _ in range(1):
# # sess.run(next_element)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
step = 0
try:
while not coord.should_stop():
# print('--------------------------------------------------------------')
label, s1, s2 = sess.run(
[label_batch, sentences_1_batch, sentences_2_batch])
step += 1
print(step, label.shape, step % 1000)
# print(sess.run(sentences_1_batch).shape, sess.run(sentences_2_batch).shape,
# sess.run(label_batch).shape)
_, loss, attraction, repulsion, dis, acc = \
sess.run([train_op, siamese.loss, siamese.attraction_loss,
siamese.repulsion_loss, siamese.distance,
siamese.accuracy],
feed_dict={
siamese.left_input: s1,
siamese.right_input: s2,
siamese.label: label,
})
log_str = "(#{0: <5} - {6}) - Loss: {1:.4f} - " \
"(a: {2:.3f} - r: {3:.3f} - " \
"d: {4:.4f}, accuracy:{5:.4f})"
print(
log_str.format(sess.run(global_step), loss,
np.mean(attraction), np.mean(repulsion),
np.mean(dis), acc,
np.mean(sess.run(label_batch))))
# TODO Dev
# if not step % 10:
# print('--------------------------------------------------------------')
# coord_dev = tf.train.Coordinator()
# threads = tf.train.start_queue_runners(coord=coord_dev, sess=sess)
# devstep = 0
# try:
# while not coord_dev.should_stop():
# label, s1, s2 = sess.run([dev_labels, dev_sentences_1, dev_sentences_2])
# devstep += 1
# print(devstep, label.shape)
# except tf.errors.OutOfRangeError:
# print("Done dev!")
# finally:
# coord.request_stop()
#
# coord.join(threads)
except tf.errors.OutOfRangeError:
print("Done training!")
finally:
coord.request_stop()
coord.join(threads)
# Save the model
timestamp = str(int(time()))
out_dir = abspath(join(curdir, "models", timestamp))
makedirs(out_dir, exist_ok=True)
with open(join(out_dir, 'parameters.txt'), 'w') as param_file:
param_file.write("Default parameters: \n")
for attr, value in sorted(FLAGS.__flags.items()):
param_file.write(" - {}={}\n".format(attr.upper(), value))
save_path = saver.save(sess, join(out_dir, "model.ckpt"))
print("Model saved in file: {}".format(save_path))
ids = os.listdir(datapath)
for id in ids:
idpath = os.path.join(datapath, id)
idfaces = os.listdir(idpath)
for id_train_face in idfaces:
if id_train_face.split('.')[-1] != 'jpg':
continue
img = cv2.imread(os.path.join(idpath, id_train_face))
img = cv2.resize(img, input_shape)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
X.append(img)
y.append(ids.index(id))
X = np.asarray(X).astype('float32')
y = np.asarray(y)
X /= 255
return X, y
input_shape = (180, 180)
num_classes = 7
x_train, y_train = load_faces_data('/data_out/siamese_faces/train')
x_test, y_test = load_faces_data('/data_out/siamese_faces/test')
siam = Siamese(x_train, y_train, x_test, y_test, input_shape, num_classes)
siam.train(epochs=5)
classifier = Classification(x_train, y_train, x_test, y_test, input_shape,
num_classes)
classifier.train(epochs=5)
print('saved split for {} and min_images={}, set make_new_split to False now'.\
format(dataset_name, min_images))
else:
print('loading existing split {}'.format(split_name))
with open(split_name, 'rb') as f:
split = pickle.load(f)
ds_train = Dataset(filenames=split.filenames_train,
data_augmentation=data_augmentation)
ds_test = Dataset(filenames=split.filenames_test)
ds_pairs_train = Dataset_pairs(ds_train, prob_same_class)
if train:
siamese = Siamese(dim_embedding, image_size, margin)
siamese.train(ds_pairs_train, learning_rate, batch_size, max_steps,
keep_prob_fc, show_loss_every_steps,
save_weights_every_steps)
print('path experiment {}'.format(siamese.path_experiment))
if compute_and_plot_embedding:
""" compute and save embeddings of the datasets.
plots can not be done inside a screen """
siamese = Siamese(dim_embedding, image_size, margin)
labels_train, points_train, images_train = siamese.inference_dataset(
ds_train, path_experiment)
labels_test, points_test, images_test = siamese.inference_dataset(
ds_test, path_experiment)
if False:
n_labels = 2 if one_hot else 1
# TEST THE SYSTEM
with tf.Graph().as_default():
label_batch, test_1_batch, test_2_batch = input_pipeline(filepath=TEST_PATH,
batch_size=1,
num_labels=n_labels,
sequence_len=seq_len,
num_epochs=1)
print(type(label_batch), type(test_1_batch), type(test_2_batch))
siamese = Siamese(seq_len,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
margin=1.0,
threshold=1.5,
fully=True,
hash_size=128)
init_op = tf.global_variables_initializer()
init_again = tf.local_variables_initializer()
predictions = {}
with tf.Session() as sess:
# Initialize variables
sess.run(init_op)
sess.run(init_again)
# Restore the model
def test_model(tf_path, model_path, flags_path):
# Import the parameters binarized
test_tfrecors = join(tf_path, 'test.tfrecords')
vocab_processor_path = join(tf_path, 'vocab.train')
vocab_processor = load_binarize_data(vocab_processor_path)
sequence_length_path = join(tf_path, 'sequence.len')
seq_len = load_binarize_data(sequence_length_path)
FLAGS = read_flags(flags_path)
# TODO Remove this from the siamese class
fully_layer = True if FLAGS.hash_size else False
# TODO this is a parameter
one_hot = False
n_labels = 2 if one_hot else 1
# TEST THE SYSTEM
with tf.Graph().as_default():
label_batch, test_1_batch, test_2_batch = input_pipeline_test(
filepath=test_tfrecors,
batch_size=1,
num_labels=n_labels,
sequence_len=seq_len,
num_epochs=1)
print(type(label_batch), type(test_1_batch), type(test_2_batch))
siamese = Siamese(sequence_length=seq_len,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
margin=FLAGS.margin)
init_op = tf.global_variables_initializer()
init_again = tf.local_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialize variables
sess.run(init_op)
sess.run(init_again)
# Restore the model
saver.restore(sess, join(model_path, "model.ckpt"))
# Create the coordinators to read the test data
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
test_sample, hits = 0, 0
try:
while not coord.should_stop():
test_1, test_2, test_label = sess.run(
[test_1_batch, test_2_batch, label_batch])
# TEST CLASSIFICATION
loss, attraction, repulsion, dis, acc = \
sess.run([siamese.loss, siamese.attr,
siamese.rep, siamese.distance,
siamese.accuracy],
feed_dict={
siamese.left_input: test_1,
siamese.right_input: test_2,
siamese.labels: test_label,
siamese.is_training: False
})
with open(join(model_path, 'test.log'), 'a') as log_file:
log_str = "(#{0: <5} - {6}) - Loss: {1:.4f} - " \
"(a: {2:.3f} - r: {3:.3f} - " \
"d: {4:.4f}, accuracy:{5:.4f})\n"
log_file.write(
log_str.format(
test_sample,
loss,
attraction[0][0],
repulsion[0][0],
dis[0],
acc,
test_label[0][0],
))
with open(join(model_path, 'distances.log'),
'a') as dist_file:
log_str = "{}\t{}\n"
dist_file.write(
log_str.format(dis[0], test_label[0][0]))
test_sample += 1
if acc == 1:
hits += 1
except tf.errors.OutOfRangeError:
print("Done testing!")
finally:
coord.request_stop()
coord.join(threads)
sess.close()
with open(join(model_path, 'results.txt'), 'w') as results_file:
results_file.write("Accuracy: {} ({}/{})".format(
hits / test_sample, hits, test_sample))
print("Results saved in: {}".format(join(model_path,
'results.txt')))
plot_distances(model_path)
def train_with_loader(train_loader, n_labels, num_channels):
model = C3D2(n_labels, num_channels)
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
if torch.cuda.is_available():
model.cuda()
checkpoint = torch.load(c.MODEL_DIR + '\\model_best.pt')
optimizer = optim.SGD(model.parameters(),
lr=c.LEARNING_RATE,
momentum=c.MOMENTUM,
weight_decay=c.WEIGHT_DECAY)
loss_criterion = Siamese(c.LAMBDA, c.M)
scheduler = CosineAnnealingLR(optimizer, T_max=10)
# scheduler = StepLR(optimizer,
# step_size=c.STEP_SIZE,
# gamma=c.GAMMA)
n_epochs = c.N_EPOCHS
train_loss = []
train_acc = []
best_accuracy = 0.0
for epoch in range(n_epochs):
train_running_loss = 0.0
# Step the lr scheduler each epoch!
scheduler.step()
total_loss = 0
start = time.time()
for i, data in enumerate(train_loader, 1):
# get the inputs
train_input, train_labels = data
y = torch.eq(train_labels[0:int(c.BATCH_SIZE / 2)],
train_labels[int(c.BATCH_SIZE /
2):c.BATCH_SIZE]).float()
if torch.cuda.is_available():
train_input, y = Variable(train_input.cuda()), Variable(
y.cuda())
else:
train_input, y = Variable(train_input), Variable(y)
# zero the parameter gradients
optimizer.zero_grad()
# forward
output1 = model(train_input[0:int(c.BATCH_SIZE / 2)],
development=False)
output2 = model(train_input[int(c.BATCH_SIZE / 2):c.BATCH_SIZE],
development=False)
# Loss
train_loss_ = loss_criterion(model, y, output1, output2)
# backward & optimization
train_loss_.backward()
optimizer.step()
train_running_loss += train_loss_.item()
if i % c.BATCH_PER_LOG == 0: # print every 2000 mini-batches
end = time.time()
total_time = end - start
print('[Epoch %d,Batch %d] loss: %.3f time: %.3f' %
(epoch + 1, i, train_running_loss / c.BATCH_PER_LOG,
total_time))
total_loss += train_running_loss / c.BATCH_PER_LOG
train_running_loss = 0.0
correct = 0
total = 1
# with torch.no_grad():
# model.eval()
# for data in train_loader:
#
# images, labels = data
# if torch.cuda.is_available():
# images = images.cuda()
# labels = labels.cuda()
#
# outputs = model(images)
# _, predicted = torch.max(outputs.data, 1)
# total += labels.size(0)
# correct += (predicted == labels).sum().item()
# model.train()
end = time.time()
total_time = end - start
print(
'Accuracy of the network: %d %%, loss: %.5f for epoch %d time %d \n'
% (100 * correct / total, total_loss, epoch + 1, total_time))
train_loss.append(total_loss)
train_acc.append(100 * correct / total)
if best_accuracy < 100 * correct / total:
if not os.path.exists(c.MODEL_DIR):
os.mkdir(c.MODEL_DIR)
print("\nBEST MODEL SO FAR")
best_accuracy = 100 * correct / total
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'accuracy': 100 * correct / total,
'optimizer': optimizer.state_dict(),
},
is_best=100 * correct / total == best_accuracy,
filename=os.path.join(c.MODEL_DIR,
'saved_' + str(epoch + 1) + '_model.pt'))
if int(epoch + 1) % c.EPOCHS_PER_SAVE == 0:
if not os.path.exists(c.MODEL_DIR):
os.mkdir(c.MODEL_DIR)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'accuracy': 100 * correct / total,
'optimizer': optimizer.state_dict(),
},
is_best=100 * correct / total == best_accuracy,
filename=os.path.join(c.MODEL_DIR,
'saved_' + str(epoch + 1) + '_model.pt'))
plot_loss_acc(train_loss, train_acc)
def main(cfg):
device = torch.device('cuda' if cfg.cuda else 'cpu')
autoenc = DeepLabv3Plus()
model = Siamese(autoenc,
in_channels=3,
n_edges=cfg.n_edges,
sp_pool_use_max=cfg.sp_pooling_max)
if (cfg.checkpoint_autoenc is not None):
print('loading checkpoint {}'.format(cfg.checkpoint_autoenc))
state_dict = torch.load(cfg.checkpoint_autoenc,
map_location=lambda storage, loc: storage)
autoenc.load_state_dict(state_dict)
elif (cfg.checkpoint_siam is not None):
print('loading checkpoint {}'.format(cfg.checkpoint_siam))
state_dict = torch.load(cfg.checkpoint_siam,
map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
autoenc.to(device)
model.to(device)
transf = iaa.Sequential([
iaa.Invert(0.5) if 'Dataset1' in 'Dataset' +
cfg.train_dir else iaa.Noop(),
iaa.SomeOf(3, [
iaa.Affine(scale={
"x": (1 - cfg.aug_scale, 1 + cfg.aug_scale),
"y": (1 - cfg.aug_scale, 1 + cfg.aug_scale)
},
rotate=(-cfg.aug_rotate, cfg.aug_rotate),
shear=(-cfg.aug_shear, cfg.aug_shear)),
iaa.SomeOf(1, [
iaa.AdditiveGaussianNoise(scale=cfg.aug_noise * 255),
iaa.GaussianBlur(sigma=(0., cfg.aug_blur)),
iaa.GammaContrast((0., cfg.aug_gamma))
]),
iaa.Fliplr(p=0.5),
iaa.Flipud(p=0.5)
]), rescale_augmenter
])
transf_normal = Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
dl_train = Loader(pjoin(cfg.in_root, 'Dataset' + cfg.train_dir),
augmentation=transf,
n_segments=cfg.n_segments_train,
delta_segments=cfg.delta_segments_train,
normalization=transf_normal)
dl_test = torch.utils.data.ConcatDataset([
Loader(pjoin(cfg.in_root, 'Dataset' + d),
augmentation=transf,
n_segments=cfg.n_segments_test,
delta_segments=cfg.delta_segments_test,
normalization=transf_normal) for d in cfg.test_dirs
])
dataloader_train = DataLoader(dl_train,
batch_size=cfg.batch_size,
sampler=SubsetRandomSampler(
cfg.n_frames_epoch * cfg.train_frames),
collate_fn=dl_train.collate_fn,
drop_last=True,
num_workers=cfg.n_workers)
dataloader_test = DataLoader(dl_test,
batch_size=cfg.batch_size,
collate_fn=dl_train.collate_fn,
sampler=torch.utils.data.RandomSampler(
dl_test,
replacement=True,
num_samples=cfg.batch_size),
num_workers=cfg.n_workers)
dataloaders = {'train': dataloader_train, 'test': dataloader_test}
d = datetime.datetime.now()
ds_dir = os.path.split('Dataset' + cfg.train_dir)[-1]
run_dir = pjoin(cfg.out_dir,
'{}_{:%Y-%m-%d_%H-%M}_{}'.format(ds_dir, d, cfg.exp_name))
if (not os.path.exists(run_dir)):
os.makedirs(run_dir)
# Save cfg
with open(pjoin(run_dir, 'cfg.yml'), 'w') as outfile:
yaml.dump(cfg.__dict__, stream=outfile, default_flow_style=False)
# convert batch to device
batch_to_device = lambda batch: {
k: v.to(device) if (isinstance(v, torch.Tensor)) else v
for k, v in batch.items()
}
optimizer = optim.SGD(params=[{
'params': model.autoenc.encoder.parameters(),
'lr': cfg.lr_autoenc
}, {
'params': model.autoenc.aspp.parameters(),
'lr': cfg.lr_autoenc
}, {
'params': model.autoenc.decoder.parameters(),
'lr': cfg.lr_siam
}, {
'params': model.linear1.parameters(),
'lr': cfg.lr_siam
}, {
'params': model.linear2.parameters(),
'lr': cfg.lr_siam
}],
momentum=cfg.momentum,
weight_decay=cfg.decay)
utls.setup_logging(run_dir)
logger = logging.getLogger('siam')
logger.info('run_dir: {}'.format(run_dir))
train(cfg, model, dataloaders, run_dir, batch_to_device, optimizer, logger)
logger.info('training siam')
def main(cfg):
device = torch.device('cuda' if cfg.cuda else 'cpu')
model = Siamese(in_channels=3).to(device)
model.eval()
transf = iaa.Sequential([
iaa.Resize(cfg.in_shape),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
rescale_augmenter
])
dl_train = Loader(pjoin(cfg.in_root, 'Dataset' + cfg.train_dir),
augmentation=transf)
dl = Loader(cfg.in_dir, augmentation=transf)
dataloader = DataLoader(dl, collate_fn=dl_train.collate_fn)
if (not os.path.exists(cfg.out_dir)):
os.makedirs(cfg.out_dir)
# convert batch to device
batch_to_device = lambda batch: {
k: v.to(device) if (isinstance(v, torch.Tensor)) else v
for k, v in batch.items()
}
out_path = pjoin(cfg.out_dir, 'sp_desc_df.p')
if (not os.path.exists(out_path)):
print('computing features on {}'.format(cfg.in_dir))
feats = []
pbar = tqdm.tqdm(total=len(dataloader))
for i, data in enumerate(dataloader):
data = batch_to_device(data)
feats_ = model.feat_extr(data['image']).cpu().detach().numpy()
labels = data['labels'][0, ...].cpu().detach().numpy()
regions = regionprops(labels)
for props in regions:
x, y = props.centroid
feats.append(
(i, props.label, x / (labels.shape[1] - 1),
y / (labels.shape[0] - 1),
feats_[labels == props.label, :].mean(axis=0).copy()))
pbar.update(1)
pbar.close()
df = pd.DataFrame(feats, columns=['frame', 'label', 'x', 'y', 'desc'])
print('Saving features to {}'.format(out_path))
df.to_pickle(out_path)
else:
print('Found features file at {}'.format(out_path))
df = pd.read_pickle(out_path)
if (cfg.do_inter_frame):
out_path = pjoin(cfg.out_dir, 'graph_inter.p')
if (not os.path.exists(out_path)):
print('computing inter-frame probabilities on {}'.format(
cfg.in_dir))
pbar = tqdm.tqdm(total=len(dataloader))
g = nx.Graph()
for f in range(np.max(df['frame'] - 1)):
df0 = df.loc[df['frame'] == f].copy(deep=False)
df1 = df.loc[df['frame'] == f + 1].copy(deep=False)
df0.columns = ['frame_0', 'label_0', 'x0', 'y0']
df1.columns = ['frame_1', 'label_1', 'x1', 'y1']
df0.loc[:, 'key'] = 1
df1.loc[:, 'key'] = 1
df_combs = pd.merge(df0, df1, on='key').drop('key', axis=1)
df_combs['rx'] = df_combs['x0'] - df_combs['x1']
df_combs['ry'] = df_combs['y0'] - df_combs['y1']
r = np.concatenate((df_combs['rx'].values.reshape(
-1, 1), df_combs['ry'].values.reshape(-1, 1)),
axis=1)
dists = np.linalg.norm(r, axis=1)
df_combs['dist'] = dists
df_combs = df_combs.loc[df_combs['dist'] < cfg.radius]
edges = [((row[1], row[2]), (row[5], row[6]))
for row in df_combs.itertuples()]
feats = [
torch.stack((df[e[0][0], e[0][1]], df[e[0][0], e[0][1]]))
for e in edges
]
feats = chunks(feats, cfg.batch_size)
probas = [
model.calc_probas(torch.stack(feats_).to(device))
for feats_ in feats
]
probas = [
p.detach().cpu().numpy().astype(np.float16) for p in probas
]
edges = [((e[0][0], e[0][1]), (e[1][0], e[1][1]),
dict(weight=p)) for e, p in zip(edges, probas)]
g.add_edges_from(edges)
pbar.update(1)
pbar.close()
print('Saving inter-frame graph to {}'.format(out_path))
with open(out_path, 'wb') as f:
pk.dump(g, f, pk.HIGHEST_PROTOCOL)
else:
print('Found inter-frame graph at {}'.format(out_path))
with open(out_path, 'rb') as f:
g = pk.load(f)
if (cfg.do_intra_frame):
out_path = pjoin(cfg.out_dir, 'graph_intra.p')
if (not os.path.exists(out_path)):
graphs = []
print('computing intra-frame probabilities on {}'.format(
cfg.in_dir))
pbar = tqdm.tqdm(total=len(dl))
for sample in dl:
g = future.graph.RAG(sample['labels'])
feats = [
torch.stack((df[e[0][0], e[0][1]], df[e[0][0], e[0][1]]))
for e in g.edges
]
feats = chunks(feats, cfg.batch_size)
probas = [
model.calc_probas(torch.stack(feats_).to(device))
for feats_ in feats
]
probas = [
p.detach().cpu().numpy().astype(np.float16) for p in probas
]
edges = [((e[0][0], e[0][1]), (e[1][0], e[1][1]),
dict(weight=p)) for e, p in zip(edges, probas)]
g.add_edges_from(edges)
graphs.append(g)
pbar.update(1)
pbar.close()
print('Saving inter-frame graph to {}'.format(out_path))
with open(out_path, 'wb') as f:
pk.dump(graphs, f, pk.HIGHEST_PROTOCOL)
else:
print('Found inter-frame graph at {}'.format(out_path))
with open(out_path, 'rb') as f:
graphs = pk.load(f)
def train():
n_samples = 40_000
learning_rate = 1e-5
num_iterations = 50_000
batch_size = 32
filenames = np.load('data/images_background_filenames.npy')
print(filenames.shape)
pairs = []
for class_filenames in filenames:
pairs.extend(itertools.combinations(class_filenames, 2))
pairs = sample(pairs, k=n_samples)
pairs = [pair + (1.0,) for pair in pairs]
for i in range(n_samples):
class_1, class_2 = choices(filenames, k=2)
im_1, im_2 = choice(class_1), choice(class_2)
pairs.append((im_1, im_2, 0.0))
shuffle(pairs)
def _parse(x1, x2, y_):
def str_to_img(s):
image_string = tf.read_file(s)
image_decoded = tf.image.decode_png(image_string)
image_resized = tf.reshape(image_decoded, (h, w, 1))
return tf.divide(image_resized, 255)
im1, im2 = map(str_to_img, [x1, x2])
return im1, im2, y_
x1, x2, y_ = map(np.array, zip(*pairs))
dataset = tf.data.Dataset.from_tensor_slices((x1, x2, y_)) \
.map(_parse, num_parallel_calls=8) \
.repeat(-1) \
.batch(batch_size) \
.prefetch(batch_size)
iterator = dataset.make_one_shot_iterator()
next_element = iterator.get_next()
siamese = Siamese()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
optimizer = tf.train.AdamOptimizer(learning_rate)
train_step = optimizer.minimize(siamese.loss)
sess.run(tf.variables_initializer(optimizer.variables()))
saver = tf.train.Saver()
min_loss = (-1, float('inf'))
for i in trange(num_iterations):
x1, x2, y_ = sess.run(next_element)
feed_dict = {
siamese.x1: x1,
siamese.x2: x2,
siamese.y_: y_,
}
_, loss_v = sess.run([train_step, siamese.loss], feed_dict=feed_dict)
assert not np.isnan(loss_v), 'Model diverged with loss = NaN'
if loss_v < min_loss[1]:
min_loss = (i, loss_v)
if i % 100 == 0:
tqdm.write(f'\nstep {i}: loss {loss_v} Minimum loss: {min_loss}')
if (i+1) % 1000 == 0:
tqdm.write('\nModel saved: {}'.format(saver.save(sess, model_path)))
print('Finished:', saver.save(sess, model_path))
def train_siamese():
"""
Performs training and evaluation of Siamese model.
First define your graph using class Siamese and its methods. Then define
necessary operations such as trainer (train_step in this case), savers
and summarizers. Finally, initialize your model within a tf.Session and
do the training.
---------------------------
How to evaluate your model:
---------------------------
On train set, it is fine to monitor loss over minibatches. On the other
hand, in order to evaluate on test set you will need to create a fixed
validation set using the data sampling function you implement for siamese
architecture. What you need to do is to iterate over all minibatches in
the validation set and calculate the average loss over all minibatches.
---------------------------------
How often to evaluate your model:
---------------------------------
- on training set every print_freq iterations
- on test set every eval_freq iterations
------------------------
Additional requirements:
------------------------
Also you are supposed to take snapshots of your model state (i.e. graph,
weights and etc.) every checkpoint_freq iterations. For this, you should
study TensorFlow's tf.train.Saver class. For more information, please
checkout:
[https://www.tensorflow.org/versions/r0.11/how_tos/variables/index.html]
"""
# Set the random seeds for reproducibility. DO NOT CHANGE.
def _check_loss(data):
loss_val = 0.
for batch in data:
x1_data, x2_data, y_data = batch
[curr_loss] = sess.run([loss],
feed_dict={
x1: x1_data,
x2: x2_data,
y: y_data
})
loss_val += curr_loss
# test_wrt.add_summary(summary_test, it)
# print(len(data))
# print(loss_val)
return loss_val / len(data)
tf.set_random_seed(42)
np.random.seed(42)
cifar10 = get_cifar_10_siamese(FLAGS.data_dir, validation_size=5000)
val_data = create_dataset_siamese(cifar10.validation, num_tuples=500)
test_data = create_dataset_siamese(cifar10.test, num_tuples=500)
#### PARAMETERS
classes = [
'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship',
'truck'
]
n_classes = len(classes)
input_data_dim = cifar10.test.images.shape[1]
#####
cnn_siamese = Siamese()
x1 = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name="x1")
x2 = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name="x2")
y = tf.placeholder(tf.float32, shape=[None], name="y")
with tf.name_scope('train_cnn'):
infs1 = cnn_siamese.inference(x1)
infs2 = cnn_siamese.inference(x2, reuse=True)
with tf.name_scope('cross-entropy-loss'):
loss = cnn_siamese.loss(infs1, infs2, y, 0.48)
merged = tf.merge_all_summaries()
opt_operation = train_step(loss)
with tf.Session() as sess:
saver = tf.train.Saver()
sess.run(tf.initialize_all_variables())
# test_loss = _check_loss(test_data)
# print("Initial Test Loss = {0:.3f}".format(test_loss))
#train_writer = tf.train.SummaryWriter(FLAGS.log_dir + "/train/", sess.graph)
#test_writer = tf.train.SummaryWriter(FLAGS.log_dir + "/test/", sess.graph)
val_losses = []
train_losses = []
for iteration in range(FLAGS.max_steps + 1):
x1_train, x2_train, y_train = cifar10.train.next_batch(
FLAGS.batch_size)
_ = sess.run([opt_operation],
feed_dict={
x1: x1_train,
x2: x2_train,
y: y_train
})
if iteration % FLAGS.print_freq == 0:
[train_loss] = sess.run([loss],
feed_dict={
x1: x1_train,
x2: x2_train,
y: y_train
})
train_losses.append(train_loss)
# train_writer.add_summary(summary_train, iteration)
print("Iteration {0:d}/{1:d}. Train Loss = {2:.6f}".format(
iteration, FLAGS.max_steps, train_loss))
if iteration % FLAGS.eval_freq == 0:
val_loss = _check_loss(val_data)
val_losses.append(val_loss)
# [test_acc, test_loss, summary_test] = sess.run([accuracy, loss, merged], feed_dict={x: x_test, y: y_test})
# test_writer.add_summary(summary_test, iteration)
print(
"Iteration {0:d}/{1:d}. Validation Loss = {2:.6f}".format(
iteration, FLAGS.max_steps, val_loss))
if iteration > 0 and iteration % FLAGS.checkpoint_freq == 0:
saver.save(sess,
FLAGS.checkpoint_dir + '/cnn_model_siamese.ckpt')
# test_loss = _check_loss(test_data)
# print("Final Test Loss = {0:.3f}".format(test_loss))
# train_writer.flush()
# test_writer.flush()
# train_writer.close()
# test_writer.close()
sess.close()
print("train_loss", train_losses)
print("val_loss", val_losses)
file = pd.read_csv('embed.csv')
# print(file['className'].unique())
means = []
for column_name in file['className'].unique():
means.append(row_mean(file, column_name))
print(np.shape(means))
# for k in range(len(means)-1):
# for j in range(k, len(means)):
# dist = np.sqrt(np.sum((means[k] - means[j])**2))
# print(k, j, dist)
siamese = Siamese()
siamese.load_model()
image = [np.array(Image.open('image_test.jpg')) / 255.0]
print(np.shape(image))
pred = siamese.test_model(image)
dists = []
for k in range(len(means)):
dist = np.sqrt(np.sum((means[k] - pred)**2))
print(k, dist)
dists.append(dist)
print('Class:', np.argmin(dists))
def train_siamese():
"""
Performs training and evaluation of Siamese model.
First define your graph using class Siamese and its methods. Then define
necessary operations such as trainer (train_step in this case), savers
and summarizers. Finally, initialize your model within a tf.Session and
do the training.
---------------------------
How to evaluate your model:
---------------------------
On train set, it is fine to monitor loss over minibatches. On the other
hand, in order to evaluate on test set you will need to create a fixed
validation set using the data sampling function you implement for siamese
architecture. What you need to do is to iterate over all minibatches in
the validation set and calculate the average loss over all minibatches.
---------------------------------
How often to evaluate your model:
---------------------------------
- on training set every print_freq iterations
- on test set every eval_freq iterations
------------------------
Additional requirements:
------------------------
Also you are supposed to take snapshots of your model state (i.e. graph,
weights and etc.) every checkpoint_freq iterations. For this, you should
study TensorFlow's tf.train.Saver class. For more information, please
checkout:
[https://www.tensorflow.org/versions/r0.11/how_tos/variables/index.html]
"""
# Set the random seeds for reproducibility. DO NOT CHANGE.
tf.set_random_seed(42)
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
########################
# Cifar10 stuff
cifar10, image_shape, num_classes = standard_cifar10_get(FLAGS)
# Placeholder variables
x1 = tf.placeholder(tf.float32,
shape=[None] + list(image_shape),
name='x1')
x2 = tf.placeholder(tf.float32,
shape=[None] + list(image_shape),
name='x2')
y = tf.placeholder(tf.float32, shape=(None), name='y')
is_training = tf.placeholder(dtype=tf.bool, shape=(), name='isTraining')
margin = tf.placeholder(tf.float32, shape=(), name='margin')
# CNN model
model = Siamese(is_training=is_training,
dropout_rate=FLAGS.dropout_rate,
save_stuff=FLAGS.save_stuff,
fc_reg_str=FLAGS.fc_reg_str)
# Get outputs of two siamese models, loss, train optimisation step
l2_out_1 = model.inference(x1)
l2_out_2 = model.inference(x2, reuse=True)
loss_no_reg, d2 = model.loss(l2_out_1, l2_out_2, y, margin)
reg_loss = sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
loss_w_reg = loss_no_reg + reg_loss
accuracy = model.accuracy(d2, y, margin)
tf.scalar_summary('loss_incl_reg', loss_w_reg)
train_op = train_step(loss_w_reg)
validation_tuples = create_dataset(
cifar10.test,
num_tuples=FLAGS.siamese_vali_ntuples,
batch_size=FLAGS.batch_size,
fraction_same=FLAGS.siamese_fraction_same)
xv1, xv2, yv = np.vstack([i[0] for i in validation_tuples]),\
np.vstack([i[1] for i in validation_tuples]),\
np.hstack([i[2] for i in validation_tuples])
num_val_chunks = 10
assert (FLAGS.siamese_vali_ntuples % num_val_chunks) == 0
chunks = range(0, xv1.shape[0], int(xv1.shape[0] / num_val_chunks)) + \
[int(FLAGS.siamese_vali_ntuples * FLAGS.batch_size)]
# Function for getting feed dicts
def get_feed(c, train=True, chunk=None, chunks=None):
if train == 'train' or train == 't':
xd1, xd2, yd = \
c.train.next_batch(FLAGS.batch_size, FLAGS.siamese_fraction_same)
return {
x1: xd1,
x2: xd2,
y: yd,
is_training: True,
margin: FLAGS.siamese_margin
}
elif train == 'vali' or train == 'v' or train == 'validation':
if chunk is None:
return {
x1: xv1,
x2: xv2,
y: yv,
is_training: False,
margin: FLAGS.siamese_margin
}
else:
st, en = chunks[chunk], chunks[chunk + 1]
return {
x1: xv1[st:en],
x2: xv2[st:en],
y: yv[st:en],
is_training: False,
margin: FLAGS.siamese_margin
}
else:
pass
# TODO Implement test set feed dict siamese
# For saving checkpoints
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialise all variables
tf.initialize_all_variables().run(session=sess)
# Merge all the summaries
merged = tf.merge_all_summaries()
if FLAGS.save_stuff:
train_writer = tf.train.SummaryWriter(FLAGS.log_dir + '/train',
sess.graph)
test_writer = tf.train.SummaryWriter(FLAGS.log_dir + '/test')
# Start training loops
for epoch in range(0, FLAGS.max_steps):
if epoch % 100 == 0:
# Print accuracy and loss on validation set
accuracies = []
losses = []
for i in range(num_val_chunks):
loss_val, acc = \
sess.run([loss_no_reg, accuracy],
get_feed(cifar10, 'vali', i, chunks))
accuracies.append(acc)
losses.append(loss_val)
# if FLAGS.save_stuff:
# test_writer.add_summary(summary, epoch)
print('\nEpoch', epoch, '\nValidation accuracy:',
np.mean(accuracies), '\nValidation loss :',
np.mean(losses))
if epoch % FLAGS.checkpoint_freq == 0:
# Save model checkpoint
if epoch > 0:
save_path = saver.save(sess, FLAGS.checkpoint_dir + \
'/epoch'+ str(epoch) + '.ckpt')
print("Model saved in file: %s" % save_path)
# Do training update
if FLAGS.save_stuff:
summary, _ = sess.run([merged, train_op],
feed_dict=get_feed(cifar10, 'train'))
train_writer.add_summary(summary, epoch)
else:
sess.run([train_op], feed_dict=get_feed(cifar10, 'train'))
# Print the final accuracy
summary, loss_val = \
sess.run([merged, loss_no_reg], get_feed(cifar10, 'vali'))
if FLAGS.save_stuff:
test_writer.add_summary(summary, epoch + 1)
print('\nFinal validation loss :', loss_val)
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True)
return loader
train_loader = create_data_loaders(ROOT_DIR,
metadata_df,
metadata_real_df,
image_size,
batch_size,
num_workers=0)
model = Siamese()
model.cuda()
import torch.optim as optim
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
for epoch in range(1):
bce_loss = 0.0
total_examples = 0
for data in tqdm(train_loader):
batch_size = data[0].shape[0]
x1 = data[0].to(gpu)
def feature_extraction():
"""
This method restores a TensorFlow checkpoint file (.ckpt) and rebuilds inference
model with restored parameters. From then on you can basically use that model in
any way you want, for instance, feature extraction, finetuning or as a submodule
of a larger architecture. However, this method should extract features from a
specified layer and store them in data files such as '.h5', '.npy'/'.npz'
depending on your preference. You will use those files later in the assignment.
Args:
[optional]
Returns:
None
"""
########################
# PUT YOUR CODE HERE #
########################
print('doing feature extraction...')
tf.reset_default_graph()
sess = tf.Session()
cifar10, image_shape, num_classes = standard_cifar10_get(FLAGS)
if FLAGS.train_model == 'siamese':
# Construct siamese graph
# Placeholder variables
x = tf.placeholder(tf.float32,
shape=[None] + list(image_shape),
name='x1')
y = tf.placeholder(tf.float32, shape=(None), name='y')
is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='isTraining')
# CNN model
model = Siamese(is_training=is_training,
dropout_rate=FLAGS.dropout_rate,
save_stuff=FLAGS.save_stuff,
fc_reg_str=FLAGS.fc_reg_str)
# Get outputs of two siamese models, loss, train optimisation step
l2 = model.inference(x)
#fc2 = model.fc2
fc2 = l2
else:
# Construct linear convnet graph
x = tf.placeholder(tf.float32,
shape=[None] + list(image_shape),
name='x')
y = tf.placeholder(tf.int32, shape=(None, num_classes), name='y')
is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='isTraining')
model = ConvNet(is_training=is_training,
dropout_rate=FLAGS.dropout_rate)
_ = model.inference(x)
fc2 = model.fc2
# Initialise all variables
tf.initialize_all_variables().run(session=sess)
# Restore checkpoint
saver = tf.train.Saver()
saver.restore(sess, FLAGS.ckpt_path + FLAGS.ckpt_file)
# Get testing data for feed dict
x_data_test, y_data_test = \
cifar10.test.images[:FLAGS.test_size], cifar10.test.labels[:FLAGS.test_size]
# Get the test set features at flatten, fc1 and fc2 layers
flatten_features_test, fc1_features_test, fc2_features_test = \
sess.run([model.flatten, model.fc1, fc2],
{x : x_data_test, y : y_data_test, is_training : False})
# Get t-SNE manifold of these features
tsne = TSNE()
manifold = tsne.fit_transform(fc2_features_test)
# Save to disk for plotting later
indices = np.arange(FLAGS.test_size)
cPickle.dump((manifold, indices),
open('manifold' + FLAGS.train_model + '.dump', 'wb'))
# Get training data for feed dict
x_data_train, y_data_train = \
cifar10.train.images[:FLAGS.train_size_lm], cifar10.train.labels[:FLAGS.train_size_lm]
# Get train set features at flatten, fc1 and fc2 layers
flatten_features_train, fc1_features_train, fc2_features_train = \
sess.run([model.flatten, model.fc1, fc2],
{x : x_data_train, y : y_data_train, is_training : False})
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import SVC
features_list = [['flat', flatten_features_train, flatten_features_train],
['fc1 ', fc1_features_train, fc1_features_test],
['fc2 ', fc2_features_train, fc2_features_test]]
for (name, features_train, features_test) in features_list:
classif = OneVsRestClassifier(SVC(kernel='linear'))
classif.fit(features_train, y_data_train)
lm_test_predictions = classif.predict(features_test)
acc = np.mean(
np.argmax(y_data_test, 1) == np.argmax(lm_test_predictions, 1))
print(name, 'accuracy =', np.round(acc * 100, 2), '%')
