def main():
open('output_summary.csv', 'w').close()
# Constants variables
NUM_TRAIN_SAMPLES = 72485
NUM_TEST_SAMPLES = 26528
# Editable variables
num_labeled_samples = 5126
num_validation_samples = 0
batch_size = 25
epochs = 200
max_learning_rate = 0.003
initial_beta1 = 0.9
final_beta1 = 0.5
checkpoint_directory = './checkpoints/PiModel'
tensorboard_logs_directory = './logs/PiModel'
# Assign it as tfe.variable since we will change it across epochs
learning_rate = tfe.Variable(max_learning_rate)
beta_1 = tfe.Variable(initial_beta1)
outputArr = np.array([])
# Download and Save Dataset in Tfrecords
#loader = SvnhLoader('./data', NUM_TRAIN_SAMPLES,
# num_validation_samples, num_labeled_samples)
#loader.download_images_and_generate_tf_record()
loader = FnLoader('./fn_data', NUM_TRAIN_SAMPLES, num_validation_samples,
num_labeled_samples)
# print ("hello")
loader.download_images_and_generate_tf_record()
#sys.exit()
# Generate data loaders
train_labeled_iterator, train_unlabeled_iterator, validation_iterator, test_iterator = loader.load_dataset(
batch_size, epochs)
#print (train_labeled_iterator)
batches_per_epoch = int(num_labeled_samples / batch_size)
batches_per_epoch_val = int(num_validation_samples / batch_size)
# sys.exit()
model = PiModel()
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta_1,
beta2=0.999)
max_unsupervised_weight = 100 * num_labeled_samples / \
(NUM_TRAIN_SAMPLES - num_validation_samples)
best_val_accuracy = 0
global_step = tf.train.get_or_create_global_step()
writer = tf.contrib.summary.create_file_writer(tensorboard_logs_directory)
writer.set_as_default()
#sys.exit()
for epoch in range(epochs):
rampdown_value = ramp_down_function(epoch, epochs)
rampup_value = ramp_up_function(epoch)
if epoch == 0:
unsupervised_weight = 0
else:
unsupervised_weight = max_unsupervised_weight * \
rampup_value
learning_rate.assign(rampup_value * rampdown_value * max_learning_rate)
beta_1.assign(rampdown_value * initial_beta1 +
(1.0 - rampdown_value) * final_beta1)
epoch_loss_avg = tfe.metrics.Mean()
epoch_accuracy = tfe.metrics.Accuracy()
epoch_loss_avg_val = tfe.metrics.Mean()
epoch_accuracy_val = tfe.metrics.Accuracy()
for batch_nr in range(batches_per_epoch):
X_labeled_train, y_labeled_train = train_labeled_iterator.get_next(
)
#print(y_labeled_train[0:20,0])
#print(y_labeled_train[0:20,1])
#print(y_labeled_train.shape)
X_unlabeled_train, _ = train_unlabeled_iterator.get_next()
loss_val, grads = pi_model_gradients(X_labeled_train,
y_labeled_train,
X_unlabeled_train, model,
unsupervised_weight)
optimizer.apply_gradients(zip(grads, model.variables),
global_step=global_step)
#sys.exit()
epoch_loss_avg(loss_val)
#print(X_labeled_train)
num_test_batches = int(NUM_TEST_SAMPLES / batch_size)
pred = model(X_labeled_train)
#sys.exit()
outputArr = np.array([])
epoch_accuracy(tf.argmax(pred, 1), tf.argmax(y_labeled_train, 1))
if (batch_nr == batches_per_epoch - 1):
for test_batch in range(num_test_batches):
X_val, y_val = test_iterator.get_next()
y_val_predictions = model(X_val, training=False)
y_pred = tf.argmax(y_val_predictions, 1)
y_true = tf.argmax(y_val, 1)
y_pred_epoch = np.asarray(y_pred)
y_true_epoch = np.asarray(y_true)
#print(y_pred, y_true)
prec_epch = sk.metrics.precision_score(
y_true_epoch, y_pred_epoch)
rec_epch = sk.metrics.recall_score(y_true_epoch,
y_pred_epoch)
f1_epch = sk.metrics.f1_score(y_true_epoch, y_pred_epoch)
epoch_loss_avg_val(
tf.losses.softmax_cross_entropy(
y_val, y_val_predictions))
epoch_accuracy_val(tf.argmax(y_val_predictions, 1),
tf.argmax(y_val, 1))
#value1 = epoch+1
#value2 = epoch_accuracy.result()
#value3 =
#value4 =
#value5 =
#value6 =
#arrResult = [epoch+1, epoch_accuracy.result(), epoch_accuracy_val, a, b, c ]
arrResult = "{:03d}, {:02.6%}, {:02.6%}, {:.4%}, {:.4%}, {:.4%} ".format(
epoch + 1, epoch_accuracy.result(), epoch_accuracy_val.result(),
prec_epch, rec_epch, f1_epch)
out = open('output_summary.csv', 'a+')
out.write(arrResult + '\n')
#writef = csv.writer(out, delimiter=' ')
#writef.writerow(arrResult)
# print("Epoch {:03d}/{:03d}: Train Loss: {:9.7f}, Train Accuracy: {:02.6%}, Validation Loss: {:9.7f}, "
# "Validation Accuracy: {:02.6%}, lr={:.9f}, unsupervised weight={:5.3f}, beta1={:.9f}".format(epoch+1,
# epochs,
# epoch_loss_avg.result(),
# epoch_accuracy.result(),
# epoch_loss_avg_val.result(),
# epoch_accuracy_val.result(),
# learning_rate.numpy(),
# unsupervised_weight,
# beta_1.numpy()))
print(
"Epoch {:03d}/{:03d}: Train Loss: {:9.7f}, Train Accuracy: {:02.6%}, lr={:.9f}, unsupervised weight={:5.3f}, beta1={:.9f}"
.format(epoch + 1, epochs, epoch_loss_avg.result(),
epoch_accuracy.result(), learning_rate.numpy(),
unsupervised_weight, beta_1.numpy()))
print(epoch_accuracy_val)
#print (epoch_accuracy.result())
# If the accuracy of validation improves save a checkpoint Best 85%
if best_val_accuracy < epoch_accuracy.result():
best_val_accuracy = epoch_accuracy.result()
checkpoint = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=global_step)
checkpoint.save(file_prefix=checkpoint_directory)
# Record summaries
#with tf.contrib.summary.record_summaries_every_n_global_steps(1):
# tf.contrib.summary.scalar('Train Loss', epoch_loss_avg.result())
# tf.contrib.summary.scalar(
# 'Train Accuracy', epoch_accuracy.result())
# tf.contrib.summary.scalar(
# 'Validation Loss', epoch_loss_avg_val.result())
# tf.contrib.summary.scalar(
# 'Validation Accuracy', epoch_accuracy_val.result())
# tf.contrib.summary.scalar(
# 'Unsupervised Weight', unsupervised_weight)
# tf.contrib.summary.scalar('Learning Rate', learning_rate.numpy())
# tf.contrib.summary.scalar('Ramp Up Function', rampup_value)
# tf.contrib.summary.scalar('Ramp Down Function', rampdown_value)
#print('\nTrain Ended! Best Validation accuracy = {}\n'.format(best_val_accuracy))
#sys.exit()
# Load the best model
root = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=tf.train.get_or_create_global_step())
root.restore(tf.train.latest_checkpoint(checkpoint_directory))
# Evaluate on the final test set
#num_test_batches = NUM_TEST_SAMPLES/batch_size
test_accuracy = tfe.metrics.Accuracy()
#recall_eval = tf.metrics.recall(y_test_predictions, y_test)
#precision_eval = tf.metrics.precision(y_test_predictions, y_test)
for test_batch in range(int(num_test_batches)):
X_test, y_test = test_iterator.get_next()
#print(y_test[0:20,1])
y_test_predictions = model(X_test, training=False)
test_accuracy(tf.argmax(y_test_predictions, 1), tf.argmax(y_test, 1))
y_pred = tf.argmax(y_test_predictions, 1)
y_true = tf.argmax(y_test, 1)
y_pred = np.asarray(y_pred)
y_true = np.asarray(y_true)
#print(y_pred, y_true)
a = sk.metrics.precision_score(y_true, y_pred)
b = sk.metrics.recall_score(y_true, y_pred)
c = sk.metrics.f1_score(y_true, y_pred)
print("Precision", a)
print("Recall", b)
print("f1_score", c)
#print ("confusion_matrix")
#print (sk.metrics.confusion_matrix(y_true, y_pred))
#fpr, tpr, tresholds = sk.metrics.roc_curve(y_true, y_pred)
#precision_eval = tf.metrics.precision(y_test_predictions, y_test)
#precision_eval = tf.contrib.metrics.precision_at_recall(tf.argmax(y_test_predictions, 1), tf.argmax(y_test, 1), 1)
print(tf.argmax(y_test_predictions))
print(tf.argmax(y_test))
#f1_score(y_test_predictions, y_test, average='macro')
print("Final Test Accuracy: {:.6%}".format(test_accuracy.result()))
def main():
# Constants variables
NUM_TRAIN_SAMPLES = 73257
NUM_TEST_SAMPLES = 26032
# Editable variables
num_labeled_samples = 3000
num_validation_samples = 1000
num_train_unlabeled_samples = NUM_TRAIN_SAMPLES - \
num_labeled_samples - num_validation_samples
batch_size = 150
epochs = 300
max_learning_rate = 0.0002 # 0.001 as recomended in the paper leads to unstable training.
initial_beta1 = 0.9
final_beta1 = 0.5
alpha = 0.6
max_unsupervised_weight = 30 * num_labeled_samples / \
(NUM_TRAIN_SAMPLES - num_validation_samples)
checkpoint_directory = './checkpoints/TemporalEnsemblingModel'
tensorboard_logs_directory = './logs/TemporalEnsemblingModel'
# Assign it as tfe.variable since we will change it across epochs
learning_rate = tfe.Variable(max_learning_rate)
beta_1 = tfe.Variable(initial_beta1)
# Download and Save Dataset in Tfrecords
loader = SvnhLoader('./data', NUM_TRAIN_SAMPLES, num_validation_samples,
num_labeled_samples)
loader.download_images_and_generate_tf_record()
# You can replace it by the real ratio (preferably with a big batch size : num_labeled_samples / num_train_unlabeled_samples
# This means that the labeled batch size will be labeled_batch_fraction * batch_size and the unlabeled batch size will be
# (1-labeled_batch_fraction) * batch_size
labeled_batch_fraction = num_labeled_samples / num_train_unlabeled_samples
batches_per_epoch = round(num_labeled_samples /
(batch_size * labeled_batch_fraction))
# Generate data loaders
train_labeled_iterator, train_unlabeled_iterator, validation_iterator, test_iterator = loader.load_dataset(
batch_size,
epochs + 1000,
labeled_batch_fraction,
1.0 - labeled_batch_fraction,
shuffle=True)
batches_per_epoch_val = int(round(num_validation_samples / batch_size))
model = PiModel()
# Paper has beta2=0.990 but I experimented decreasing it a little bit (as recomended in the paper) and it led
# to more stable training
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta_1,
beta2=0.980)
best_val_accuracy = 0
global_step = tf.train.get_or_create_global_step()
writer = tf.contrib.summary.create_file_writer(tensorboard_logs_directory)
writer.set_as_default()
# Ensemble predictions - the first samples of the array are for the labeled samples
# and the remaining ones are for the unlabeled samples.
# The Z and z are the notation used in the paper
Z = np.zeros((NUM_TRAIN_SAMPLES, 10))
z = np.zeros((NUM_TRAIN_SAMPLES, 10))
# variable needed if you use a batch ratio different than the true ratio
sample_epoch = np.zeros((NUM_TRAIN_SAMPLES, 1))
for epoch in range(epochs):
rampdown_value = ramp_down_function(epoch, epochs)
# In the paper the authors use 80 as the epoch with max rampup_value
rampup_value = ramp_up_function(epoch, 40)
if epoch == 0:
unsupervised_weight = 0
else:
unsupervised_weight = max_unsupervised_weight * \
rampup_value
learning_rate.assign(rampup_value * rampdown_value * max_learning_rate)
beta_1.assign(rampdown_value * initial_beta1 +
(1.0 - rampdown_value) * final_beta1)
epoch_loss_avg = tfe.metrics.Mean()
epoch_accuracy = tfe.metrics.Accuracy()
epoch_loss_avg_val = tfe.metrics.Mean()
epoch_accuracy_val = tfe.metrics.Accuracy()
for batch_nr in range(batches_per_epoch):
X_labeled_train, y_labeled_train, labeled_indexes = train_labeled_iterator.get_next(
)
X_unlabeled_train, _, unlabeled_indexes = train_unlabeled_iterator.get_next(
)
# We need to correct labeled samples indexes (in Z the first num_labeled_samples samples are for ensemble labeled predictions)
current_ensemble_indexes = np.concatenate([
labeled_indexes.numpy(),
unlabeled_indexes.numpy() + num_labeled_samples
])
current_ensemble_targets = z[current_ensemble_indexes]
current_outputs, loss_val, grads = temporal_ensembling_gradients(
X_labeled_train, y_labeled_train, X_unlabeled_train, model,
unsupervised_weight, current_ensemble_targets)
optimizer.apply_gradients(zip(grads, model.variables),
global_step=global_step)
epoch_loss_avg(loss_val)
epoch_accuracy(tf.argmax(model(X_labeled_train), 1),
tf.argmax(y_labeled_train, 1))
epoch_loss_avg(loss_val)
epoch_accuracy(tf.argmax(model(X_labeled_train), 1),
tf.argmax(y_labeled_train, 1))
Z[current_ensemble_indexes, :] = alpha * \
Z[current_ensemble_indexes, :] + (1-alpha) * current_outputs
z[current_ensemble_indexes, :] = Z[current_ensemble_indexes, :] * \
(1. / (1. - alpha **
(sample_epoch[current_ensemble_indexes] + 1)))
sample_epoch[current_ensemble_indexes] += 1
if (batch_nr == batches_per_epoch - 1):
for batch_val_nr in range(batches_per_epoch_val):
X_val, y_val, _ = validation_iterator.get_next()
y_val_predictions = model(X_val, training=False)
epoch_loss_avg_val(
tf.losses.softmax_cross_entropy(
y_val, y_val_predictions))
epoch_accuracy_val(tf.argmax(y_val_predictions, 1),
tf.argmax(y_val, 1))
print(
"Epoch {:03d}/{:03d}: Train Loss: {:9.7f}, Train Accuracy: {:02.6%}, Validation Loss: {:9.7f}, "
"Validation Accuracy: {:02.6%}, lr={:.9f}, unsupervised weight={:5.3f}, beta1={:.9f}"
.format(epoch + 1, epochs, epoch_loss_avg.result(),
epoch_accuracy.result(), epoch_loss_avg_val.result(),
epoch_accuracy_val.result(), learning_rate.numpy(),
unsupervised_weight, beta_1.numpy()))
# If the accuracy of validation improves save a checkpoint
if best_val_accuracy < epoch_accuracy_val.result():
best_val_accuracy = epoch_accuracy_val.result()
checkpoint = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=global_step)
checkpoint.save(file_prefix=checkpoint_directory)
# Record summaries
with tf.contrib.summary.record_summaries_every_n_global_steps(1):
tf.contrib.summary.scalar('Train Loss', epoch_loss_avg.result())
tf.contrib.summary.scalar('Train Accuracy',
epoch_accuracy.result())
tf.contrib.summary.scalar('Validation Loss',
epoch_loss_avg_val.result())
tf.contrib.summary.histogram('Z',
tf.convert_to_tensor(Z),
step=global_step)
tf.contrib.summary.histogram('z',
tf.convert_to_tensor(z),
step=global_step)
tf.contrib.summary.scalar('Validation Accuracy',
epoch_accuracy_val.result())
tf.contrib.summary.scalar('Unsupervised Weight',
unsupervised_weight)
tf.contrib.summary.scalar('Learning Rate', learning_rate.numpy())
tf.contrib.summary.scalar('Ramp Up Function', rampup_value)
tf.contrib.summary.scalar('Ramp Down Function', rampdown_value)
print('\nTrain Ended! Best Validation accuracy = {}\n'.format(
best_val_accuracy))
# Load the best model
root = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=tf.train.get_or_create_global_step())
root.restore(tf.train.latest_checkpoint(checkpoint_directory))
# Evaluate on the final test set
num_test_batches = math.ceil(NUM_TEST_SAMPLES / batch_size)
test_accuracy = tfe.metrics.Accuracy()
for test_batch in range(num_test_batches):
X_test, y_test, _ = test_iterator.get_next()
y_test_predictions = model(X_test, training=False)
test_accuracy(tf.argmax(y_test_predictions, 1), tf.argmax(y_test, 1))
print("Final Test Accuracy: {:.6%}".format(test_accuracy.result()))
def main():
# Constants variables
NUM_TRAIN_SAMPLES = 73257
NUM_TEST_SAMPLES = 26032
# Editable variables
num_labeled_samples = 200
num_validation_samples = 50
num_train_unlabeled_samples = NUM_TRAIN_SAMPLES - num_labeled_samples - num_validation_samples
# num_train_unlabeled_samples=训练样本总数-标记样本-验证集
# 迭代300代，每代25个批次
batch_size = 10 # 每个batch中数据量大小
epochs = 10
max_learning_rate = 0.001
initial_beta1 = 0.9
final_beta1 = 0.5
checkpoint_directory = './checkpoints/PiModel'
tensorboard_logs_directory = './logs/PiModel'
# Assign it as tfe.variable since we will change it across epochs：要在各个时期更改，相当于变量
learning_rate = tfe.Variable(max_learning_rate)
beta_1 = tfe.Variable(initial_beta1) # tf.Variable的变量
# Download and Save Dataset in Tfrecords：创建tf格式的文件
loader = SvnhLoader('./data', NUM_TRAIN_SAMPLES, num_validation_samples,
num_labeled_samples) # 最先调用的init函数
loader.download_images_and_generate_tf_record()
# Generate data loaders 相当于是一个操作
train_labeled_iterator, train_unlabeled_iterator, validation_iterator, test_iterator = loader.load_dataset(
batch_size, epochs)
# 每个epoch中有多少个batch（标记数据集/验证集）
batches_per_epoch = int(num_labeled_samples / batch_size)
batches_per_epoch_val = int(num_validation_samples / batch_size)
# 初始化model对象，优化器，最大的无监督权重
model = PiModel()
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta_1,
beta2=0.990) # Adam优化
max_unsupervised_weight = 100 * num_labeled_samples / (
NUM_TRAIN_SAMPLES - num_validation_samples)
best_val_accuracy = 0
# 创建全局步长张量
global_step = tf.train.get_or_create_global_step()
writer = tf.contrib.summary.create_file_writer(
tensorboard_logs_directory) # 写日志
writer.set_as_default()
for epoch in range(epochs):
rampdown_value = ramp_down_function(epoch, epochs)
rampup_value = ramp_up_function(epoch)
if epoch == 0:
unsupervised_weight = 0
else:
unsupervised_weight = max_unsupervised_weight * rampup_value
learning_rate.assign(rampup_value * rampdown_value * max_learning_rate)
beta_1.assign(rampdown_value * initial_beta1 +
(1.0 - rampdown_value) * final_beta1)
epoch_loss_avg = tfe.metrics.Mean()
epoch_accuracy = tfe.metrics.Accuracy()
epoch_loss_avg_val = tfe.metrics.Mean()
epoch_accuracy_val = tfe.metrics.Accuracy() # 返回的是tfe.metrics.Mean对象
for batch_nr in range(batches_per_epoch):
X_labeled_train, y_labeled_train, _ = train_labeled_iterator.get_next(
) # 数据代入点,返回一个张量，大小为batch_size*32*32*3
X_unlabeled_train, _, _ = train_unlabeled_iterator.get_next()
loss_val, grads = pi_model_gradients(X_labeled_train,
y_labeled_train,
X_unlabeled_train, model,
unsupervised_weight)
optimizer.apply_gradients(zip(grads, model.variables),
global_step=global_step)
epoch_loss_avg(loss_val)
epoch_accuracy(tf.argmax(model(X_labeled_train), 1),
tf.argmax(y_labeled_train, 1))
if (batch_nr == batches_per_epoch - 1): # 在最后一个batch中验证准确率
for batch_val_nr in range(batches_per_epoch_val):
X_val, y_val, _ = validation_iterator.get_next()
y_val_predictions = model(X_val, training=False)
epoch_loss_avg_val(
tf.losses.softmax_cross_entropy(
y_val, y_val_predictions))
epoch_accuracy_val(tf.argmax(y_val_predictions, 1),
tf.argmax(y_val, 1))
print(
"Epoch {:03d}/{:03d}: Train Loss: {:9.7f}, Train Accuracy: {:02.6%}, Validation Loss: {:9.7f}, "
"Validation Accuracy: {:02.6%}, lr={:.9f}, unsupervised weight={:5.3f}, beta1={:.9f}"
.format(epoch + 1, epochs, epoch_loss_avg.result(),
epoch_accuracy.result(), epoch_loss_avg_val.result(),
epoch_accuracy_val.result(), learning_rate.numpy(),
unsupervised_weight, beta_1.numpy()))
# If the accuracy of validation improves save a checkpoint Best 85% 记录checkpoint
if best_val_accuracy < epoch_accuracy_val.result():
best_val_accuracy = epoch_accuracy_val.result()
checkpoint = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=global_step)
checkpoint.save(file_prefix=checkpoint_directory)
# Record summaries
with tf.contrib.summary.record_summaries_every_n_global_steps(1):
tf.contrib.summary.scalar('Train Loss', epoch_loss_avg.result())
tf.contrib.summary.scalar('Train Accuracy',
epoch_accuracy.result())
tf.contrib.summary.scalar('Validation Loss',
epoch_loss_avg_val.result())
tf.contrib.summary.scalar('Validation Accuracy',
epoch_accuracy_val.result())
tf.contrib.summary.scalar('Unsupervised Weight',
unsupervised_weight)
tf.contrib.summary.scalar('Learning Rate', learning_rate.numpy())
tf.contrib.summary.scalar('Ramp Up Function', rampup_value)
tf.contrib.summary.scalar('Ramp Down Function', rampdown_value)
print('\nTrain Ended! Best Validation accuracy = {}\n'.format(
best_val_accuracy)) # 以上都是训练
# Load the best model 接下来是测试集
root = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=tf.train.get_or_create_global_step())
root.restore(tf.train.latest_checkpoint(checkpoint_directory))
# Evaluate on the final test set 扎到最优的模型，给出测试准确率
num_test_batches = math.ceil(NUM_TEST_SAMPLES / batch_size)
test_accuracy = tfe.metrics.Accuracy()
for test_batch in range(num_test_batches):
X_test, y_test, _ = test_iterator.get_next()
y_test_predictions = model(X_test, training=False)
test_accuracy(tf.argmax(y_test_predictions, 1), tf.argmax(y_test, 1))
print("Final Test Accuracy: {:.6%}".format(test_accuracy.result()))
def main():
# Constants variables
# NUM_TRAIN_SAMPLES = 73257
# NUM_TEST_SAMPLES = 26032
NUM_TRAIN_SAMPLES = 73257
# Editable variables
num_labeled_samples = 1000
num_validation_samples = 1000
NUM_TEST_SAMPLES = NUM_TRAIN_SAMPLES - num_labeled_samples - num_validation_samples
num_train_unlabeled_samples = NUM_TRAIN_SAMPLES - \
num_labeled_samples - num_validation_samples
batch_size = 25
epochs = 10
max_learning_rate = 0.001
initial_beta1 = 0.9
final_beta1 = 0.5
checkpoint_directory = './checkpoints/PiModel'
tensorboard_logs_directory = './logs/PiModel'
# Assign it as tfe.variable since we will change it across epochs
learning_rate = tfe.Variable(max_learning_rate)
beta_1 = tfe.Variable(initial_beta1)
# Download and Save Dataset in Tfrecords
loader = SvnhLoader('./data', NUM_TRAIN_SAMPLES,
num_validation_samples, num_labeled_samples)
loader.download_images_and_generate_tf_record()
# Generate data loaders
train_labeled_iterator, train_unlabeled_iterator, validation_iterator, test_iterator = loader.load_dataset(
batch_size, epochs)
batches_per_epoch = int(num_labeled_samples/batch_size)
batches_per_epoch_val = int(num_validation_samples / batch_size)
model = PiModel()
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate, beta1=beta_1, beta2=0.990)
max_unsupervised_weight = 100 * num_labeled_samples / \
(NUM_TRAIN_SAMPLES - num_validation_samples)
best_val_accuracy = 0
global_step = tf.train.get_or_create_global_step()
writer = tf.contrib.summary.create_file_writer(tensorboard_logs_directory)
writer.set_as_default()
if train:
for epoch in range(epochs):
rampdown_value = ramp_down_function(epoch, epochs)
rampup_value = ramp_up_function(epoch)
if epoch == 0:
unsupervised_weight = 0
else:
unsupervised_weight = max_unsupervised_weight * \
rampup_value
learning_rate.assign(rampup_value * rampdown_value * max_learning_rate)
beta_1.assign(rampdown_value * initial_beta1 +
(1.0 - rampdown_value) * final_beta1)
epoch_loss_avg = tfe.metrics.Mean()
epoch_accuracy = tfe.metrics.Accuracy()
epoch_loss_avg_val = tfe.metrics.Mean()
epoch_accuracy_val = tfe.metrics.Accuracy()
for batch_nr in range(batches_per_epoch):
X_labeled_train, y_labeled_train, _ = train_labeled_iterator.get_next()
X_unlabeled_train, _, _ = train_unlabeled_iterator.get_next()
loss_val, grads = pi_model_gradients(X_labeled_train, y_labeled_train, X_unlabeled_train,
model, unsupervised_weight)
optimizer.apply_gradients(zip(grads, model.variables),
global_step=global_step)
epoch_loss_avg(loss_val)
epoch_accuracy(
tf.argmax(model(X_labeled_train), 1), tf.argmax(y_labeled_train, 1))
if (batch_nr == batches_per_epoch - 1):
for batch_val_nr in range(batches_per_epoch_val):
X_val, y_val, _ = validation_iterator.get_next()
y_val_predictions = model(X_val, training=False)
epoch_loss_avg_val(tf.losses.softmax_cross_entropy(
y_val, y_val_predictions))
epoch_accuracy_val(
tf.argmax(y_val_predictions, 1), tf.argmax(y_val, 1))
print("Epoch {:03d}/{:03d}: Train Loss: {:9.7f}, Train Accuracy: {:02.6%}, Validation Loss: {:9.7f}, "
"Validation Accuracy: {:02.6%}, lr={:.9f}, unsupervised weight={:5.3f}, beta1={:.9f}".format(epoch+1,
epochs,
epoch_loss_avg.result(),
epoch_accuracy.result(),
epoch_loss_avg_val.result(),
epoch_accuracy_val.result(),
learning_rate.numpy(),
unsupervised_weight,
beta_1.numpy()))
# If the accuracy of validation improves save a checkpoint Best 85%
if best_val_accuracy < epoch_accuracy_val.result():
best_val_accuracy = epoch_accuracy_val.result()
checkpoint = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=global_step)
checkpoint.save(file_prefix=checkpoint_directory)
# Record summaries
with tf.contrib.summary.record_summaries_every_n_global_steps(1):
tf.contrib.summary.scalar('Train Loss', epoch_loss_avg.result())
tf.contrib.summary.scalar(
'Train Accuracy', epoch_accuracy.result())
tf.contrib.summary.scalar(
'Validation Loss', epoch_loss_avg_val.result())
tf.contrib.summary.scalar(
'Validation Accuracy', epoch_accuracy_val.result())
tf.contrib.summary.scalar(
'Unsupervised Weight', unsupervised_weight)
tf.contrib.summary.scalar('Learning Rate', learning_rate.numpy())
tf.contrib.summary.scalar('Ramp Up Function', rampup_value)
tf.contrib.summary.scalar('Ramp Down Function', rampdown_value)
print('\nTrain Ended! Best Validation accuracy = {}\n'.format(best_val_accuracy))
# Load the best model
root = tfe.Checkpoint(optimizer=optimizer,
model=model,
optimizer_step=tf.train.get_or_create_global_step())
root.restore(tf.train.latest_checkpoint(checkpoint_directory))
all_features = []
labels = []
for batch_nr in range(batches_per_epoch):
X_labeled_train, y_labeled_train, _ = train_labeled_iterator.get_next()
# print(X_labeled_train.shape)
all_features.append(X_labeled_train)
labels.append(y_labeled_train)
for batch_val_nr in range(batches_per_epoch_val):
X_val, y_val, _ = validation_iterator.get_next()
all_features.append(X_val)
labels.append(y_val)
# Evaluate on the final test set
num_test_batches = math.ceil(NUM_TEST_SAMPLES/batch_size)
print("Num of test batches is ", num_test_batches)
test_accuracy = tfe.metrics.Accuracy()
for test_batch in range(num_test_batches):
X_test, y_test, _ = test_iterator.get_next()
print(X_test.shape)
y_test_predictions = model(X_test, training=False)
labels.append(y_test_predictions)
test_accuracy(tf.argmax(y_test_predictions, 1), tf.argmax(y_test, 1))
print("Final Test Accuracy: {:.6%}".format(test_accuracy.result()))