def default(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
method = "default"
# log list
training_log = []
num_train_images = input_reader.num_train_images
num_test_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.height, input_reader.width, input_reader.depth]
# batch pathcer
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label)
test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label)
config = tf.ConfigProto(allow_soft_placement = True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config = config) as sess:
train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True)
test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False)
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label)
elif model_type == "DenseNet-40-12":
model = DenseNet(40, 12, image_shape, num_label)
elif model_type == "DenseNet-10-12":
model = DenseNet(10, 12, image_shape, num_label)
elif model_type == "VGG-19":
model = VGG19(image_shape, num_label)
# register training operations on Trainer class
trainer = Trainer(model)
trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type)
trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op()
trainer.init_op = tf.global_variables_initializer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord)
# load data set in main memory
train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels)
test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels)
# noise injection
train_batch_patcher.set_noise(noise_rate, noise_type)
######################## main methodology for training #######################
sess.run(trainer.init_op)
#train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=True))
training(sess, total_epochs, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method, training_log=training_log)
##############################################################################
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/log.csv", "w")
for text in training_log:
f.write(text + "\n")
f.close()
def recency_bias(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up, queue_size, s_es, epochs, log_dir="log", restore=False, pre_trained=10):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
method = "ada-boundary"
# log list
training_log = []
#training_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, training loss, training error, test loss, test error")
start_time = time.time()
num_train_images = input_reader.num_train_images
num_test_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.resize_height, input_reader.resize_width, input_reader.depth]
# batch pathcer
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label)
test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label)
# online self label correcter
sampler = adaptive_sampler.Sampler(num_train_images, num_label, queue_size, s_es, epochs)
config = tf.ConfigProto(allow_soft_placement = True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config = config) as sess:
train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True, normalize = False)
test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False, normalize = False)
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label)
elif model_type == "ResNet-50":
model = ResNet(50, image_shape, num_label)
# register training operations on Trainer class
trainer = Trainer(model)
trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type)
trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op()
trainer.init_op = tf.global_variables_initializer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord)
# load data set in main memory
train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels)
test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels)
######################## main methodology for training #######################
# 1. classify clean samples using prediction certainty
print("history length : ", queue_size)
if queue_size > warm_up + pre_trained:
sys.exit(-1)
print("warm-up parameter error!")
# init params
if restore:
start_time = time.time()
saver = tf.train.Saver()
file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(init_lr) + "_e=" + str(pre_trained) + "/"
minus_start_time = 0
with open(file_dir + "log.csv") as f:
for line in f:
print(line, end="")
training_log.append(line)
minus_start_time = line.split(",")[1]
start_time = start_time - float(minus_start_time)
saver.restore(sess, file_dir + "model.ckpt")
# load history
with open(file_dir + "table.txt") as f:
index = 0
for line in f:
temp_counter = line.split("\t")[0]
temp_history = line.split("\t")[1].split(",")
sampler.update_counters[index] = int(temp_counter.strip())
for j in range(len(temp_history)-1):
sampler.all_predictions[index][j%queue_size] = float(temp_history[j%queue_size].strip())
index = index + 1
else:
pre_trained = 0
sess.run(trainer.init_op)
# warm-up
print("warm_up phase")
training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained, "warm-up", start_time, sampler=sampler, training_log=training_log)
print("ada-boundary phase")
# self online correction mechanism
training(sess, total_epochs-warm_up-pre_trained, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained + warm_up, method, start_time, sampler=sampler, training_log=training_log)
##############################################################################
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/log.csv", "w")
for text in training_log:
f.write(text + "\n")
f.close()
def OnlineBatch(gpu_id,
input_reader,
model_type,
training_epochs,
batch_size,
lr_boundaries,
lr_values,
optimizer_type,
warm_up_period,
s_e=20.0,
pretrain=0,
log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
update_method = "online_bs"
text_log = []
text_log.append(
"epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error"
)
num_train_images = input_reader.num_train_images
num_val_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.width, input_reader.height, input_reader.depth]
train_batch_patcher = online_patcher.BatchPatcher(
num_train_images,
batch_size,
num_label,
s_e=s_e,
update_method=update_method)
validation_batch_patcher = online_patcher.BatchPatcher(
num_val_images, batch_size, num_label, update_method=update_method)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config=config) as sess:
t_ids, t_images, t_labels = input_reader.data_read(batch_size,
train=True)
v_ids, v_images, v_labels = input_reader.data_read(batch_size,
train=False)
# Model Graph Construction ###########################################################################
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label,
batch_size, batch_size)
elif model_type == "WideResNet16-8":
model = WideResNet(16, 8, image_shape, num_label,
batch_size, batch_size)
train_loss_op, train_accuracy_op, train_op, _, train_distance_op = model.build_train_op(
lr_boundaries, lr_values, optimizer_type)
test_loss_op, test_accuracy_op, test_xentropy_op = model.build_test_op(
)
# Data load in memeory ###############################################################################
print("start to load data set.")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
train_batch_patcher.bulk_load_in_memory(
sess, t_ids, t_images, t_labels)
validation_batch_patcher.bulk_load_in_memory(
sess, v_ids, v_images, v_labels)
start_time = time.time()
# Model Initialization ###########################################################################
# init params: we share the initial epochs. See paper.
if pretrain != 0:
start_time = time.time()
saver = tf.train.Saver()
file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(
lr_values[0]) + "_e=" + str(pretrain) + "/"
minus_start_time = 0
with open(file_dir + "text_log.csv") as f:
for line in f:
print(line, end="")
text_log.append(line)
minus_start_time = line.split(",")[1]
start_time = start_time - float(minus_start_time)
saver.restore(sess, file_dir + "model.ckpt")
print("shared weight is successfully loaded")
else:
sess.run(tf.global_variables_initializer())
# Traning
for epoch in range(pretrain, training_epochs):
if epoch < warm_up_period:
is_warm_up = True
else:
is_warm_up = False
######################
# after 1 epoch, update loss table and calculate importance based on the paper
# "https://arxiv.org/pdf/1511.06343.pdf", Title: Online Batch Selection For Faster Training of Neural Networks######################################
loss_map = {}
for i in range(
train_batch_patcher.num_iters_per_epoch):
ids, images, labels = train_batch_patcher.get_init_mini_batch(
i)
xentropy = sess.run(
test_xentropy_op,
feed_dict={
model.test_image_placeholder: images,
model.test_label_placeholder: labels
})
for j in range(len(ids)):
sample_id = ids[j]
if sample_id in loss_map:
continue
else:
loss_map[sample_id] = xentropy[j]
if len(loss_map) != input_reader.num_train_images:
print(
"error: size of loss map is not equal to the size of data set."
)
train_batch_patcher.update_importance_table_based_on_online_bs(
loss_map)
###########################################################################################################################################################
avg_mini_loss = 0.0
avg_mini_acc = 0.0
for i in range(train_batch_patcher.num_iters_per_epoch):
# Next batch depends on the method: online batch
ids, images, labels = train_batch_patcher.get_next_mini_batch(
num_of_sample=batch_size, is_warm_up=is_warm_up)
# Parameter updates
mini_loss, mini_acc, _, distance = sess.run(
[
train_loss_op, train_accuracy_op, train_op,
train_distance_op
],
feed_dict={
model.train_image_placeholder: images,
model.train_label_placeholder: labels
})
avg_mini_loss += mini_loss
avg_mini_acc += mini_acc
avg_mini_loss /= train_batch_patcher.num_iters_per_epoch
avg_mini_acc /= train_batch_patcher.num_iters_per_epoch
# (2) Compute training loss and error
avg_train_loss = 0.0
avg_train_acc = 0.0
for i in range(train_batch_patcher.num_iters_per_epoch):
ids, images, labels = train_batch_patcher.get_init_mini_batch(
i)
train_loss, train_acc = sess.run(
[test_loss_op, test_accuracy_op],
feed_dict={
model.test_image_placeholder: images,
model.test_label_placeholder: labels
})
avg_train_loss += train_loss
avg_train_acc += train_acc
avg_train_loss /= train_batch_patcher.num_iters_per_epoch
avg_train_acc /= train_batch_patcher.num_iters_per_epoch
# (3) Validation (or test) loss and error
avg_val_loss = 0.0
avg_val_acc = 0.0
for i in range(
validation_batch_patcher.num_iters_per_epoch):
ids, images, labels = validation_batch_patcher.get_init_mini_batch(
i)
val_loss, val_acc = sess.run(
[test_loss_op, test_accuracy_op],
feed_dict={
model.test_image_placeholder: images,
model.test_label_placeholder: labels
})
avg_val_loss += val_loss
avg_val_acc += val_acc
avg_val_loss /= validation_batch_patcher.num_iters_per_epoch
avg_val_acc /= validation_batch_patcher.num_iters_per_epoch
# Log Writing ####################################################################################
cur_lr = sess.run(model.learning_rate)
print((epoch + 1), ", ", int(time.time() - start_time),
", ", cur_lr, ", ", avg_mini_loss, ", ",
(1.0 - avg_mini_acc), ", ", avg_train_loss, ", ",
(1.0 - avg_train_acc), ", ", avg_val_loss, ", ",
(1.0 - avg_val_acc))
text_log.append(
str(epoch + 1) + ", " +
str(int(time.time() - start_time)) + ", " +
str(cur_lr) + ", " + str(avg_mini_loss) + ", " +
str(1.0 - avg_mini_acc) + ", " + str(avg_train_loss) +
", " + str(1.0 - avg_train_acc) + ", " +
str(avg_val_loss) + ", " + str(1.0 - avg_val_acc))
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/text_log.csv", "w")
for text in text_log:
f.write(text + "\n")
f.close()
def RandomBatch(gpu_id, input_reader, model_type, training_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, pretrain=0, log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
text_log = []
text_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error")
num_train_images = input_reader.num_train_images
num_val_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.width, input_reader.height, input_reader.depth]
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label)
validation_batch_patcher = patcher.BatchPatcher(num_val_images, batch_size, num_label)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config=config) as sess:
# Input Graph Generation #############################################################################
t_ids, t_images, t_labels = input_reader.data_read(batch_size, train=True)
v_ids, v_images, v_labels = input_reader.data_read(batch_size, train=False)
# Model Graph Construction ###########################################################################
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label, batch_size, batch_size)
elif model_type == "WideResNet16-8":
model = WideResNet(16, 8, image_shape, num_label, batch_size, batch_size)
train_loss_op, train_accuracy_op, train_op, _, train_distance_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type)
test_loss_op, test_accuracy_op, _ = model.build_test_op()
# Data load in memeory ###############################################################################
print("start to load data set.")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
train_batch_patcher.bulk_load_in_memory(sess, t_ids, t_images, t_labels)
validation_batch_patcher.bulk_load_in_memory(sess, v_ids, v_images, v_labels)
start_time = time.time()
# Model Initialization ###########################################################################
# init params: we share the initial epochs. See paper.
if pretrain != 0:
start_time = time.time()
saver = tf.train.Saver()
file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(lr_values[0]) + "_e=" + str(pretrain) + "/"
minus_start_time = 0
with open(file_dir + "text_log.csv") as f:
for line in f:
print(line, end="")
text_log.append(line)
minus_start_time = line.split(",")[1]
start_time = start_time - float(minus_start_time)
saver.restore(sess, file_dir + "model.ckpt")
print("shared weight is successfully loaded")
else:
sess.run(tf.global_variables_initializer())
# Traing Process #####################################################################################
for epoch in range(pretrain, training_epochs):
# (1) Mini-batch loss and error along with netowrk updates
avg_mini_loss = 0.0
avg_mini_acc = 0.0
for i in range(train_batch_patcher.num_iters_per_epoch):
# if is_warm_up = True, then select next batch samples uniformly at random
ids, images, labels = train_batch_patcher.get_next_mini_batch(num_of_sample=batch_size, is_warm_up=True)
mini_loss, mini_acc, _, distance = sess.run([train_loss_op, train_accuracy_op, train_op, train_distance_op], feed_dict={model.train_image_placeholder: images, model.train_label_placeholder: labels})
avg_mini_loss += mini_loss
avg_mini_acc += mini_acc
avg_mini_loss /= train_batch_patcher.num_iters_per_epoch
avg_mini_acc /= train_batch_patcher.num_iters_per_epoch
# (2) Compute training loss and error
avg_train_loss = 0.0
avg_train_acc = 0.0
for i in range(train_batch_patcher.num_iters_per_epoch):
ids, images, labels = train_batch_patcher.get_init_mini_batch(i)
train_loss, train_acc = sess.run([test_loss_op, test_accuracy_op], feed_dict={model.test_image_placeholder: images, model.test_label_placeholder: labels})
avg_train_loss += train_loss
avg_train_acc += train_acc
avg_train_loss /= train_batch_patcher.num_iters_per_epoch
avg_train_acc /= train_batch_patcher.num_iters_per_epoch
# (3) Validation (or test) loss and error
avg_val_loss = 0.0
avg_val_acc = 0.0
for i in range(validation_batch_patcher.num_iters_per_epoch):
ids, images, labels = validation_batch_patcher.get_init_mini_batch(i)
val_loss, val_acc = sess.run([test_loss_op, test_accuracy_op], feed_dict={model.test_image_placeholder: images, model.test_label_placeholder: labels})
avg_val_loss += val_loss
avg_val_acc += val_acc
avg_val_loss /= validation_batch_patcher.num_iters_per_epoch
avg_val_acc /= validation_batch_patcher.num_iters_per_epoch
# Log Writing ####################################################################################
cur_lr = sess.run(model.learning_rate)
print((epoch + 1), ", ", int(time.time() - start_time) ,", ", cur_lr, ", ", avg_mini_loss, ", ", (1.0-avg_mini_acc), ", ", avg_train_loss, ", ", (1.0-avg_train_acc), ", ", avg_val_loss, ", ", (1.0-avg_val_acc))
text_log.append(str(epoch + 1) + ", " + str(int(time.time() - start_time)) + ", " + str(cur_lr) + ", " + str(avg_mini_loss) + ", " + str(1.0-avg_mini_acc) + ", " + str(avg_train_loss) + ", " + str(1.0-avg_train_acc) + ", " + str(avg_val_loss) + ", " + str(1.0-avg_val_acc))
coord.request_stop()
coord.join(threads)
sess.close()
# Log Flushing
f = open(log_dir + "/text_log.csv", "w")
for text in text_log:
f.write(text + "\n")
f.close()
def coteaching(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, T_k=15, log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# log list
text_log = []
text_log.append("epcoh, learning rate, training loss (network1), training error (network1), training loss (network2), training error (network2), "
"test loss (network1), test error (network1), test loss (network2), test error (network2)\n")
num_train_images = input_reader.num_train_images
num_test_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.height, input_reader.width, input_reader.depth]
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label)
test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label)
config = tf.ConfigProto(allow_soft_placement = True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config = config) as sess:
train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True)
test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False)
if model_type == "DenseNet-25-12":
model1 = DenseNet(25, 12, image_shape, num_label, scope='network1')
model2 = DenseNet(25, 12, image_shape, num_label, scope='network2')
elif model_type == "DenseNet-40-12":
model1 = DenseNet(40, 12, image_shape, num_label, scope='network1')
model2 = DenseNet(40, 12, image_shape, num_label, scope='network2')
elif model_type == "DenseNet-10-12":
model1 = DenseNet(10, 12, image_shape, num_label, scope='network1')
model2 = DenseNet(10, 12, image_shape, num_label, scope='network2')
elif model_type == "VGG-19":
model1 = VGG19(image_shape, num_label, scope='network1')
model2 = VGG19(image_shape, num_label, scope='network2')
# register training operations on Trainer class
trainer1 = Trainer(model1)
trainer1.train_loss_op, trainer1.train_accuracy_op, trainer1.train_op, trainer1.train_xentropy_op, trainer1.train_prob_op = model1.build_train_op(lr_boundaries, lr_values, optimizer_type)
trainer1.test_loss_op, trainer1.test_accuracy_op, trainer1.test_xentropy_op, trainer1.test_prob_op = model1.build_test_op()
trainer2 = Trainer(model2)
trainer2.train_loss_op, trainer2.train_accuracy_op, trainer2.train_op, trainer2.train_xentropy_op, trainer2.train_prob_op = model2.build_train_op(lr_boundaries, lr_values, optimizer_type)
trainer2.test_loss_op, trainer2.test_accuracy_op, trainer2.test_xentropy_op, trainer2.test_prob_op = model2.build_test_op()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord)
# load data set in main memory
train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels)
test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels)
# give noise on data set
train_batch_patcher.set_noise(noise_rate, noise_type)
######################## main methodology for training #######################
sess.run(tf.global_variables_initializer())
# for model 1
for epoch in range(total_epochs):
ratio = 1.0 - noise_rate * np.fmin(1.0, float(float(epoch) + 1.0) / float(T_k))
# print("Clean sample ratio: ", ratio)
for i in range(train_batch_patcher.num_iters_per_epoch):
ids, images, labels = train_batch_patcher.get_next_random_mini_batch(batch_size)
# losses of model 1
xentropy_array1 = sess.run(trainer1.train_xentropy_op, feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels})
# losses of model 2
xentropy_array2 = sess.run(trainer2.train_xentropy_op, feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels})
# choose R(T)% small-loss instances
ids_for_2, images_for_2, labels_for_2 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array1, ratio)
ids_for_1, images_for_1, labels_for_1 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array2, ratio)
# model update
_, _ = sess.run([trainer1.train_op, trainer2.train_op], feed_dict={trainer1.model.train_image_placeholder: images_for_1, trainer1.model.train_label_placeholder: labels_for_1, trainer2.model.train_image_placeholder: images_for_2, trainer2.model.train_label_placeholder: labels_for_2})
# inference and test error for logs
# test
avg_val_loss_1 = 0.0
avg_val_acc_1 = 0.0
avg_val_loss_2 = 0.0
avg_val_acc_2 = 0.0
for i in range(test_batch_patcher.num_iters_per_epoch):
ids, images, labels = test_batch_patcher.get_init_mini_batch(i)
val_loss_1, val_acc_1 = sess.run([trainer1.test_loss_op, trainer1.test_accuracy_op], feed_dict={trainer1.model.test_image_placeholder: images, trainer1.model.test_label_placeholder: labels})
val_loss_2, val_acc_2 = sess.run([trainer2.test_loss_op, trainer2.test_accuracy_op], feed_dict={trainer2.model.test_image_placeholder: images, trainer2.model.test_label_placeholder: labels})
avg_val_loss_1 += val_loss_1
avg_val_acc_1 += val_acc_1
avg_val_loss_2 += val_loss_2
avg_val_acc_2 += val_acc_2
avg_val_loss_1 /= test_batch_patcher.num_iters_per_epoch
avg_val_acc_1 /= test_batch_patcher.num_iters_per_epoch
avg_val_loss_2 /= test_batch_patcher.num_iters_per_epoch
avg_val_acc_2 /= test_batch_patcher.num_iters_per_epoch
# Inference
avg_train_loss_1 = 0.0
avg_train_acc_1 = 0.0
avg_train_loss_2 = 0.0
avg_train_acc_2 = 0.0
for i in range(train_batch_patcher.num_iters_per_epoch):
ids, images, labels = train_batch_patcher.get_init_mini_batch(i)
imp_loss_1, imp_acc_1 = sess.run([trainer1.train_loss_op, trainer1.train_accuracy_op], feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels})
imp_loss_2, imp_acc_2 = sess.run([trainer2.train_loss_op, trainer2.train_accuracy_op], feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels})
avg_train_loss_1 += imp_loss_1
avg_train_acc_1 += imp_acc_1
avg_train_loss_2 += imp_loss_2
avg_train_acc_2 += imp_acc_2
avg_train_loss_1 /= train_batch_patcher.num_iters_per_epoch
avg_train_acc_1 /= train_batch_patcher.num_iters_per_epoch
avg_train_loss_2 /= train_batch_patcher.num_iters_per_epoch
avg_train_acc_2 /= train_batch_patcher.num_iters_per_epoch
cur_lr = sess.run(trainer1.model.learning_rate)
print((epoch + 1), ", ", cur_lr, ", ", avg_train_loss_1, ", ", avg_train_acc_1,
", ", avg_train_loss_2, ", ", avg_train_acc_2,
", ", avg_val_loss_1, ", ", avg_val_acc_1,
", ", avg_val_loss_2, ", ", avg_val_acc_2)
text_log.append(str(epoch + 1) + ", " + str(cur_lr) + ", " + str(avg_train_loss_1) + ", " + str(1.0 - avg_train_acc_1)
+ ", " + str(avg_train_loss_2) + ", " + str(1.0 - avg_train_acc_2)
+ ", " + str(avg_val_loss_1) + ", " + str(1.0 - avg_val_acc_1)
+ ", " + str(avg_val_loss_2) + ", " + str(1.0 - avg_val_acc_2))
##############################################################################
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/log.csv", "w")
for text in text_log:
f.write(text + "\n")
f.close()
def selfie(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, threshold=0.05, queue_size=15, restart=3, log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
method = "selfie"
# log list
training_log = []
training_log.append("epcoh, learning rate, training loss, training error, test loss, test error\n")
correction_log = []
num_train_images = input_reader.num_train_images
num_test_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.height, input_reader.width, input_reader.depth]
# batch pathcer
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label)
test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label)
# corrector to classify clean and refurbishable samples
correcter = self_correcter.Correcter(num_train_images, num_label, queue_size, threshold, loaded_data=train_batch_patcher.loaded_data)
config = tf.ConfigProto(allow_soft_placement = True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config = config) as sess:
train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True)
test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False)
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label)
elif model_type == "DenseNet-40-12":
model = DenseNet(40, 12, image_shape, num_label)
elif model_type == "DenseNet-10-12":
model = DenseNet(10, 12, image_shape, num_label)
elif model_type == "VGG-19":
model = VGG19(image_shape, num_label)
# register training operations on Trainer class
trainer = Trainer(model)
trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type)
trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op()
trainer.init_op = tf.global_variables_initializer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord)
# load data set in main memory
train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels)
test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels)
# clean data set => simulated noisy data set
train_batch_patcher.set_noise(noise_rate, noise_type)
######################## main methodology for training #######################
# exception handler
if queue_size > warm_up:
sys.exit(-1)
print("warm-up parameter error!")
# start training
train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=True), correction_log)
for r in range(restart+1):
print("# run: ", (r+1))
sess.run(trainer.init_op)
# warm-up periods
training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", correcter=correcter, training_log=training_log)
# selfie
training(sess, total_epochs-warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, noise_rate=noise_rate, correcter=correcter, training_log=training_log, correction_log=correction_log)
# corrected confusion matrix
train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=False), correction_log)
# clean prediction history for next run
correcter.predictions_clear()
# recalculate noise rate of data based on refurbished samples
noise_rate = np.fmin(noise_rate, correcter.compute_new_noise_ratio())
print("new noise rate: ", noise_rate)
correction_log.append("new noise rate, " + str(noise_rate))
##############################################################################
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/log.csv", "w")
for text in training_log:
f.write(text + "\n")
f.close()
f = open(log_dir + "/detail_log.csv", "w")
for text in correction_log:
f.write(text + "\n")
f.close()
def selfie(gpu_id,
input_reader,
model_type,
total_epochs,
batch_size,
lr_boundaries,
lr_values,
optimizer_type,
noise_rate,
noise_type,
warm_up,
threshold,
queue_size,
restart=2,
log_dir="log"):
if not os.path.exists(log_dir):
os.makedirs(log_dir)
method = "selfie"
# log list
training_log = []
num_train_images = input_reader.num_train_images
num_test_images = input_reader.num_val_images
num_label = input_reader.num_classes
image_shape = [input_reader.height, input_reader.width, input_reader.depth]
# batch pathcer
train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size,
num_label)
test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size,
num_label)
# online self label correcter
correcter = self_correcter.Correcter(
num_train_images,
num_label,
queue_size,
threshold,
loaded_data=train_batch_patcher.loaded_data)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.visible_device_list = str(gpu_id)
config.gpu_options.allow_growth = True
graph = tf.Graph()
with graph.as_default():
with tf.device('/gpu:' + str(gpu_id)):
with tf.Session(config=config) as sess:
train_ids, train_images, train_labels = input_reader.data_read(
batch_size, train=True)
test_ids, test_images, test_labels = input_reader.data_read(
batch_size, train=False)
if model_type == "DenseNet-25-12":
model = DenseNet(25, 12, image_shape, num_label)
elif model_type == "DenseNet-40-12":
model = DenseNet(40, 12, image_shape, num_label)
elif model_type == "DenseNet-10-12":
model = DenseNet(10, 12, image_shape, num_label)
elif model_type == "VGG-19":
model = VGG19(image_shape, num_label)
# register training operations on Trainer class
trainer = Trainer(model)
trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(
lr_boundaries, lr_values, optimizer_type)
trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op(
)
trainer.init_op = tf.global_variables_initializer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# load data set in main memory
train_batch_patcher.bulk_load_in_memory(
sess, train_ids, train_images, train_labels)
test_batch_patcher.bulk_load_in_memory(sess, test_ids,
test_images,
test_labels)
# noise injection
train_batch_patcher.set_noise(noise_rate, noise_type)
#train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix())
######################## main methodology for training #######################
# 1. classify clean samples using prediction certainty
print("history length : ", queue_size)
if queue_size > warm_up:
sys.exit(-1)
print("warm-up parameter error!")
for i in range(restart + 1):
print("Restart: ", i)
sess.run(trainer.init_op)
# warm-up
training(sess,
warm_up,
batch_size,
train_batch_patcher,
test_batch_patcher,
trainer,
0,
method="warm-up",
correcter=correcter,
training_log=training_log)
# self online correction mechanism
training(sess,
total_epochs - warm_up,
batch_size,
train_batch_patcher,
test_batch_patcher,
trainer,
warm_up,
method=method,
noise_rate=noise_rate,
correcter=correcter,
training_log=training_log)
correcter.predictions_clear()
##############################################################################
coord.request_stop()
coord.join(threads)
sess.close()
f = open(log_dir + "/log.csv", "w")
for text in training_log:
f.write(text + "\n")
f.close()