def recency_bias(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up, queue_size, s_es, epochs, log_dir="log", restore=False, pre_trained=10): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "ada-boundary" # log list training_log = [] #training_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, training loss, training error, test loss, test error") start_time = time.time() num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.resize_height, input_reader.resize_width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # online self label correcter sampler = adaptive_sampler.Sampler(num_train_images, num_label, queue_size, s_es, epochs) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True, normalize = False) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False, normalize = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "ResNet-50": model = ResNet(50, image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) ######################## main methodology for training ####################### # 1. classify clean samples using prediction certainty print("history length : ", queue_size) if queue_size > warm_up + pre_trained: sys.exit(-1) print("warm-up parameter error!") # init params if restore: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(init_lr) + "_e=" + str(pre_trained) + "/" minus_start_time = 0 with open(file_dir + "log.csv") as f: for line in f: print(line, end="") training_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") # load history with open(file_dir + "table.txt") as f: index = 0 for line in f: temp_counter = line.split("\t")[0] temp_history = line.split("\t")[1].split(",") sampler.update_counters[index] = int(temp_counter.strip()) for j in range(len(temp_history)-1): sampler.all_predictions[index][j%queue_size] = float(temp_history[j%queue_size].strip()) index = index + 1 else: pre_trained = 0 sess.run(trainer.init_op) # warm-up print("warm_up phase") training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained, "warm-up", start_time, sampler=sampler, training_log=training_log) print("ada-boundary phase") # self online correction mechanism training(sess, total_epochs-warm_up-pre_trained, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained + warm_up, method, start_time, sampler=sampler, training_log=training_log) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()
def active_bias(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, smoothness=0.2, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "active_bias" # log list training_log = [] training_log.append( "epcoh, learning rate, training loss, training error, test loss, test error\n" ) num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # online self label correcter sampler = active_bias_sampler.Sampler(num_train_images, num_label, smoothness) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: train_ids, train_images, train_labels = input_reader.data_read( batch_size, train=True) train_ids, test_images, test_labels = input_reader.data_read( batch_size, train=False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op( ) trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory( sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # give noise on data set train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### sess.run(trainer.init_op) # warm-up train_batch_patcher.print_transition_matrix( train_batch_patcher.get_current_noise_matrix(entire=True)) training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", sampler=sampler, training_log=training_log) # active learning (sample loss re-weighting) training(sess, total_epochs - warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, sampler=sampler, training_log=training_log) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()
def ActiveBias(gpu_id, input_reader, model_type, training_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up_period, smoothness=0.02, pretrain=0, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) text_log = [] text_log.append( "epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error" ) num_train_images = input_reader.num_train_images num_val_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.width, input_reader.height, input_reader.depth] train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) validation_batch_patcher = patcher.BatchPatcher(num_val_images, batch_size, num_label) sampler = active_bias_sampler.Sampler(num_train_images, num_label, smoothness) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: # Input Graph Generation ############################################################################# t_ids, t_images, t_labels = input_reader.data_read(batch_size, train=True) v_ids, v_images, v_labels = input_reader.data_read(batch_size, train=False) # Model Graph Construction ########################################################################### if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label, batch_size, batch_size) elif model_type == "WideResNet16-8": model = WideResNet(16, 8, image_shape, num_label, batch_size, batch_size) train_loss_op, train_accuracy_op, train_op, train_xentropy_op, train_prob_op, train_distance_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) test_loss_op, test_accuracy_op, _ = model.build_test_op() # Data load in memeory ############################################################################### print("start to load data set.") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) train_batch_patcher.bulk_load_in_memory( sess, t_ids, t_images, t_labels) validation_batch_patcher.bulk_load_in_memory( sess, v_ids, v_images, v_labels) start_time = time.time() # Model Initialization ########################################################################### # init params: we share the initial epochs. See paper. if pretrain != 0: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str( lr_values[0]) + "_e=" + str(pretrain) + "/" minus_start_time = 0 with open(file_dir + "text_log.csv") as f: for line in f: print(line, end="") text_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") print("shared weight is successfully loaded") else: sess.run(tf.global_variables_initializer()) # Traing Process ##################################################################################### for epoch in range(pretrain, training_epochs): # (1) Mini-batch loss and error along with netowrk updates avg_mini_loss = 0.0 avg_mini_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): # if is_warm_up = True, then select next batch samples uniformly at random ids, images, labels = train_batch_patcher.get_next_mini_batch( num_of_sample=batch_size) if epoch < warm_up_period: weights = sampler.compute_sample_weights( ids, uniform=True) else: weights = sampler.compute_sample_weights(ids) mini_loss, mini_acc, _, softmax_matrix, distance = sess.run( [ train_loss_op, train_accuracy_op, train_op, train_prob_op, train_distance_op ], feed_dict={ model.train_image_placeholder: images, model.train_label_placeholder: labels, model.train_weight_placeholder: weights }) sampler.async_update_probability_matrix( ids, labels, softmax_matrix) avg_mini_loss += mini_loss avg_mini_acc += mini_acc avg_mini_loss /= train_batch_patcher.num_iters_per_epoch avg_mini_acc /= train_batch_patcher.num_iters_per_epoch # (2) Compute training loss and error avg_train_loss = 0.0 avg_train_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch( i) train_loss, train_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_train_loss += train_loss avg_train_acc += train_acc avg_train_loss /= train_batch_patcher.num_iters_per_epoch avg_train_acc /= train_batch_patcher.num_iters_per_epoch # (3) Validation (or test) loss and error avg_val_loss = 0.0 avg_val_acc = 0.0 for i in range( validation_batch_patcher.num_iters_per_epoch): ids, images, labels = validation_batch_patcher.get_init_mini_batch( i) val_loss, val_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_val_loss += val_loss avg_val_acc += val_acc avg_val_loss /= validation_batch_patcher.num_iters_per_epoch avg_val_acc /= validation_batch_patcher.num_iters_per_epoch # Log Writing #################################################################################### cur_lr = sess.run(model.learning_rate) print((epoch + 1), ", ", int(time.time() - start_time), ", ", cur_lr, ", ", avg_mini_loss, ", ", (1.0 - avg_mini_acc), ", ", avg_train_loss, ", ", (1.0 - avg_train_acc), ", ", avg_val_loss, ", ", (1.0 - avg_val_acc)) text_log.append( str(epoch + 1) + ", " + str(int(time.time() - start_time)) + ", " + str(cur_lr) + ", " + str(avg_mini_loss) + ", " + str(1.0 - avg_mini_acc) + ", " + str(avg_train_loss) + ", " + str(1.0 - avg_train_acc) + ", " + str(avg_val_loss) + ", " + str(1.0 - avg_val_acc)) coord.request_stop() coord.join(threads) sess.close() # Log Flushing f = open(log_dir + "/text_log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def coteaching(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, T_k=15, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) # log list text_log = [] text_log.append("epcoh, learning rate, training loss (network1), training error (network1), training loss (network2), training error (network2), " "test loss (network1), test error (network1), test loss (network2), test error (network2)\n") num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False) if model_type == "DenseNet-25-12": model1 = DenseNet(25, 12, image_shape, num_label, scope='network1') model2 = DenseNet(25, 12, image_shape, num_label, scope='network2') elif model_type == "DenseNet-40-12": model1 = DenseNet(40, 12, image_shape, num_label, scope='network1') model2 = DenseNet(40, 12, image_shape, num_label, scope='network2') elif model_type == "DenseNet-10-12": model1 = DenseNet(10, 12, image_shape, num_label, scope='network1') model2 = DenseNet(10, 12, image_shape, num_label, scope='network2') elif model_type == "VGG-19": model1 = VGG19(image_shape, num_label, scope='network1') model2 = VGG19(image_shape, num_label, scope='network2') # register training operations on Trainer class trainer1 = Trainer(model1) trainer1.train_loss_op, trainer1.train_accuracy_op, trainer1.train_op, trainer1.train_xentropy_op, trainer1.train_prob_op = model1.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer1.test_loss_op, trainer1.test_accuracy_op, trainer1.test_xentropy_op, trainer1.test_prob_op = model1.build_test_op() trainer2 = Trainer(model2) trainer2.train_loss_op, trainer2.train_accuracy_op, trainer2.train_op, trainer2.train_xentropy_op, trainer2.train_prob_op = model2.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer2.test_loss_op, trainer2.test_accuracy_op, trainer2.test_xentropy_op, trainer2.test_prob_op = model2.build_test_op() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # give noise on data set train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### sess.run(tf.global_variables_initializer()) # for model 1 for epoch in range(total_epochs): ratio = 1.0 - noise_rate * np.fmin(1.0, float(float(epoch) + 1.0) / float(T_k)) # print("Clean sample ratio: ", ratio) for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_next_random_mini_batch(batch_size) # losses of model 1 xentropy_array1 = sess.run(trainer1.train_xentropy_op, feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels}) # losses of model 2 xentropy_array2 = sess.run(trainer2.train_xentropy_op, feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels}) # choose R(T)% small-loss instances ids_for_2, images_for_2, labels_for_2 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array1, ratio) ids_for_1, images_for_1, labels_for_1 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array2, ratio) # model update _, _ = sess.run([trainer1.train_op, trainer2.train_op], feed_dict={trainer1.model.train_image_placeholder: images_for_1, trainer1.model.train_label_placeholder: labels_for_1, trainer2.model.train_image_placeholder: images_for_2, trainer2.model.train_label_placeholder: labels_for_2}) # inference and test error for logs # test avg_val_loss_1 = 0.0 avg_val_acc_1 = 0.0 avg_val_loss_2 = 0.0 avg_val_acc_2 = 0.0 for i in range(test_batch_patcher.num_iters_per_epoch): ids, images, labels = test_batch_patcher.get_init_mini_batch(i) val_loss_1, val_acc_1 = sess.run([trainer1.test_loss_op, trainer1.test_accuracy_op], feed_dict={trainer1.model.test_image_placeholder: images, trainer1.model.test_label_placeholder: labels}) val_loss_2, val_acc_2 = sess.run([trainer2.test_loss_op, trainer2.test_accuracy_op], feed_dict={trainer2.model.test_image_placeholder: images, trainer2.model.test_label_placeholder: labels}) avg_val_loss_1 += val_loss_1 avg_val_acc_1 += val_acc_1 avg_val_loss_2 += val_loss_2 avg_val_acc_2 += val_acc_2 avg_val_loss_1 /= test_batch_patcher.num_iters_per_epoch avg_val_acc_1 /= test_batch_patcher.num_iters_per_epoch avg_val_loss_2 /= test_batch_patcher.num_iters_per_epoch avg_val_acc_2 /= test_batch_patcher.num_iters_per_epoch # Inference avg_train_loss_1 = 0.0 avg_train_acc_1 = 0.0 avg_train_loss_2 = 0.0 avg_train_acc_2 = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch(i) imp_loss_1, imp_acc_1 = sess.run([trainer1.train_loss_op, trainer1.train_accuracy_op], feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels}) imp_loss_2, imp_acc_2 = sess.run([trainer2.train_loss_op, trainer2.train_accuracy_op], feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels}) avg_train_loss_1 += imp_loss_1 avg_train_acc_1 += imp_acc_1 avg_train_loss_2 += imp_loss_2 avg_train_acc_2 += imp_acc_2 avg_train_loss_1 /= train_batch_patcher.num_iters_per_epoch avg_train_acc_1 /= train_batch_patcher.num_iters_per_epoch avg_train_loss_2 /= train_batch_patcher.num_iters_per_epoch avg_train_acc_2 /= train_batch_patcher.num_iters_per_epoch cur_lr = sess.run(trainer1.model.learning_rate) print((epoch + 1), ", ", cur_lr, ", ", avg_train_loss_1, ", ", avg_train_acc_1, ", ", avg_train_loss_2, ", ", avg_train_acc_2, ", ", avg_val_loss_1, ", ", avg_val_acc_1, ", ", avg_val_loss_2, ", ", avg_val_acc_2) text_log.append(str(epoch + 1) + ", " + str(cur_lr) + ", " + str(avg_train_loss_1) + ", " + str(1.0 - avg_train_acc_1) + ", " + str(avg_train_loss_2) + ", " + str(1.0 - avg_train_acc_2) + ", " + str(avg_val_loss_1) + ", " + str(1.0 - avg_val_acc_1) + ", " + str(avg_val_loss_2) + ", " + str(1.0 - avg_val_acc_2)) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def active_bias(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up, smoothness=0.02, log_dir="log", restore=False, pre_trained=10): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "active_bias" # log list training_log = [] #training_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, training loss, training error, test loss, test error") start_time = time.time() num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.resize_height, input_reader.resize_width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) sampler = active_bias_sampler.Sampler(num_train_images, num_label, smoothness) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True, normalize = False) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False, normalize = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "ResNet-50": model = ResNet(50, image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) ######################## main methodology for training ####################### if restore: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(init_lr) + "_e=" + str(pre_trained) + "/" minus_start_time = 0 with open(file_dir + "log.csv") as f: for line in f: print(line, end="") training_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") else: pre_trained = 0 sess.run(trainer.init_op) # warm-up training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained, "warm-up", sampler, start_time, training_log=training_log) # active learning (sample loss re-weighting) training(sess, total_epochs-warm_up-pre_trained, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained + warm_up, method, sampler, start_time, training_log=training_log) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()
def selfie(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, threshold=0.05, queue_size=15, restart=3, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "selfie" # log list training_log = [] training_log.append("epcoh, learning rate, training loss, training error, test loss, test error\n") correction_log = [] num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # corrector to classify clean and refurbishable samples correcter = self_correcter.Correcter(num_train_images, num_label, queue_size, threshold, loaded_data=train_batch_patcher.loaded_data) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # clean data set => simulated noisy data set train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### # exception handler if queue_size > warm_up: sys.exit(-1) print("warm-up parameter error!") # start training train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=True), correction_log) for r in range(restart+1): print("# run: ", (r+1)) sess.run(trainer.init_op) # warm-up periods training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", correcter=correcter, training_log=training_log) # selfie training(sess, total_epochs-warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, noise_rate=noise_rate, correcter=correcter, training_log=training_log, correction_log=correction_log) # corrected confusion matrix train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=False), correction_log) # clean prediction history for next run correcter.predictions_clear() # recalculate noise rate of data based on refurbished samples noise_rate = np.fmin(noise_rate, correcter.compute_new_noise_ratio()) print("new noise rate: ", noise_rate) correction_log.append("new noise rate, " + str(noise_rate)) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close() f = open(log_dir + "/detail_log.csv", "w") for text in correction_log: f.write(text + "\n") f.close()
def selfie(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, threshold, queue_size, restart=2, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "selfie" # log list training_log = [] num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # online self label correcter correcter = self_correcter.Correcter( num_train_images, num_label, queue_size, threshold, loaded_data=train_batch_patcher.loaded_data) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: train_ids, train_images, train_labels = input_reader.data_read( batch_size, train=True) test_ids, test_images, test_labels = input_reader.data_read( batch_size, train=False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op( ) trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory( sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # noise injection train_batch_patcher.set_noise(noise_rate, noise_type) #train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix()) ######################## main methodology for training ####################### # 1. classify clean samples using prediction certainty print("history length : ", queue_size) if queue_size > warm_up: sys.exit(-1) print("warm-up parameter error!") for i in range(restart + 1): print("Restart: ", i) sess.run(trainer.init_op) # warm-up training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", correcter=correcter, training_log=training_log) # self online correction mechanism training(sess, total_epochs - warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, noise_rate=noise_rate, correcter=correcter, training_log=training_log) correcter.predictions_clear() ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()