def _build_model(self): """选择采用哪种卷积网络""" if self.network == CNNNetwork.CNN5: x = CNN5(model_conf=self.model_conf, inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.CNNX: x = CNNX(model_conf=self.model_conf, inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.ResNetTiny: x = ResNetTiny(model_conf=self.model_conf, inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.ResNet50: x = ResNet50(model_conf=self.model_conf, inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.DenseNet: x = DenseNet(model_conf=self.model_conf, inputs=self.inputs, utils=self.utils).build() else: raise ValueError('This cnn neural network is not supported at this time.') """选择采用哪种循环网络""" # time_major = True: [max_time_step, batch_size, num_classes] tf.compat.v1.logging.info("CNN Output: {}".format(x.get_shape())) self.seq_len = tf.fill([tf.shape(x)[0]], tf.shape(x)[1], name="seq_len") # self.labels_len = tf.fill([BATCH_SIZE], 12, name="labels_len") if self.recurrent == RecurrentNetwork.NoRecurrent: self.recurrent_network_builder = None elif self.recurrent == RecurrentNetwork.LSTM: self.recurrent_network_builder = LSTM(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.BiLSTM: self.recurrent_network_builder = BiLSTM(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.GRU: self.recurrent_network_builder = GRU(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.BiGRU: self.recurrent_network_builder = BiGRU(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.LSTMcuDNN: self.recurrent_network_builder = LSTMcuDNN(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.BiLSTMcuDNN: self.recurrent_network_builder = BiLSTMcuDNN(model_conf=self.model_conf, inputs=x, utils=self.utils) elif self.recurrent == RecurrentNetwork.GRUcuDNN: self.recurrent_network_builder = GRUcuDNN(model_conf=self.model_conf, inputs=x, utils=self.utils) else: raise ValueError('This recurrent neural network is not supported at this time.') logits = self.recurrent_network_builder.build() if self.recurrent_network_builder else x if self.recurrent_network_builder and self.model_conf.loss_func != LossFunction.CTC: raise ValueError('CTC loss must use recurrent neural network.') """输出层,根据Loss函数区分""" with tf.keras.backend.name_scope('output'): if self.model_conf.loss_func == LossFunction.CTC: self.outputs = FullConnectedRNN(model_conf=self.model_conf, mode=self.mode, outputs=logits).build() elif self.model_conf.loss_func == LossFunction.CrossEntropy: self.outputs = FullConnectedCNN(model_conf=self.model_conf, mode=self.mode, outputs=logits).build() return self.outputs
def recency_bias(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up, queue_size, s_es, epochs, log_dir="log", restore=False, pre_trained=10): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "ada-boundary" # log list training_log = [] #training_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, training loss, training error, test loss, test error") start_time = time.time() num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.resize_height, input_reader.resize_width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # online self label correcter sampler = adaptive_sampler.Sampler(num_train_images, num_label, queue_size, s_es, epochs) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True, normalize = False) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False, normalize = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "ResNet-50": model = ResNet(50, image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) ######################## main methodology for training ####################### # 1. classify clean samples using prediction certainty print("history length : ", queue_size) if queue_size > warm_up + pre_trained: sys.exit(-1) print("warm-up parameter error!") # init params if restore: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(init_lr) + "_e=" + str(pre_trained) + "/" minus_start_time = 0 with open(file_dir + "log.csv") as f: for line in f: print(line, end="") training_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") # load history with open(file_dir + "table.txt") as f: index = 0 for line in f: temp_counter = line.split("\t")[0] temp_history = line.split("\t")[1].split(",") sampler.update_counters[index] = int(temp_counter.strip()) for j in range(len(temp_history)-1): sampler.all_predictions[index][j%queue_size] = float(temp_history[j%queue_size].strip()) index = index + 1 else: pre_trained = 0 sess.run(trainer.init_op) # warm-up print("warm_up phase") training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained, "warm-up", start_time, sampler=sampler, training_log=training_log) print("ada-boundary phase") # self online correction mechanism training(sess, total_epochs-warm_up-pre_trained, batch_size, train_batch_patcher, test_batch_patcher, trainer, pre_trained + warm_up, method, start_time, sampler=sampler, training_log=training_log) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()
def _build_model(self): if self.network == CNNNetwork.CNN5: x = CNN5(inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.ResNet: x = ResNet50(inputs=self.inputs, utils=self.utils).build() elif self.network == CNNNetwork.DenseNet: x = DenseNet(inputs=self.inputs, utils=self.utils).build() else: tf.logging.error( 'This cnn neural network is not supported at this time.') sys.exit(-1) # time_major = True: [max_time_step, batch_size, num_classes] # time_major = False: [batch_size, max_time_step, num_classes] tf.logging.info("CNN Output: {}".format(x.get_shape())) self.seq_len = tf.fill([tf.shape(x)[0]], tf.shape(x)[1], name="seq_len") if self.recurrent == RecurrentNetwork.LSTM: recurrent_network_builder = LSTM(self.utils, x, self.seq_len) elif self.recurrent == RecurrentNetwork.BLSTM: recurrent_network_builder = BLSTM(self.utils, x, self.seq_len) elif self.recurrent == RecurrentNetwork.GRU: recurrent_network_builder = GRU(x, self.seq_len) elif self.recurrent == RecurrentNetwork.SRU: recurrent_network_builder = SRU(x, self.seq_len) elif self.recurrent == RecurrentNetwork.BSRU: recurrent_network_builder = BSRU(self.utils, x, self.seq_len) else: tf.logging.error( 'This recurrent neural network is not supported at this time.') sys.exit(-1) outputs = recurrent_network_builder.build() # Reshaping to apply the same weights over the time_steps outputs = tf.reshape(outputs, [-1, NUM_HIDDEN * 2]) with tf.variable_scope('output'): # tf.Variable weight_out = tf.get_variable( name='weight', shape=[ outputs.get_shape()[1] if self.network == CNNNetwork.ResNet else NUM_HIDDEN * 2, NUM_CLASSES ], dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), # initializer=tf.truncated_normal_initializer(stddev=0.1), # initializer=tf.glorot_normal_initializer(), # initializer=tf.glorot_uniform_initializer(), ) biases_out = tf.get_variable(name='biases', shape=[NUM_CLASSES], dtype=tf.float32, initializer=tf.constant_initializer( value=0, dtype=tf.float32)) # [batch_size * max_timesteps, num_classes] logits = tf.matmul(outputs, weight_out) + biases_out # Reshaping back to the original shape logits = tf.reshape(logits, [tf.shape(x)[0], -1, NUM_CLASSES]) # Time major predict = tf.transpose(logits, (1, 0, 2), "predict") self.predict = predict
def default(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "default" # log list training_log = [] num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # noise injection train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### sess.run(trainer.init_op) #train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=True)) training(sess, total_epochs, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method, training_log=training_log) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()
def OnlineBatch(gpu_id, input_reader, model_type, training_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, warm_up_period, s_e=20.0, pretrain=0, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) update_method = "online_bs" text_log = [] text_log.append( "epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error" ) num_train_images = input_reader.num_train_images num_val_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.width, input_reader.height, input_reader.depth] train_batch_patcher = online_patcher.BatchPatcher( num_train_images, batch_size, num_label, s_e=s_e, update_method=update_method) validation_batch_patcher = online_patcher.BatchPatcher( num_val_images, batch_size, num_label, update_method=update_method) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: t_ids, t_images, t_labels = input_reader.data_read(batch_size, train=True) v_ids, v_images, v_labels = input_reader.data_read(batch_size, train=False) # Model Graph Construction ########################################################################### if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label, batch_size, batch_size) elif model_type == "WideResNet16-8": model = WideResNet(16, 8, image_shape, num_label, batch_size, batch_size) train_loss_op, train_accuracy_op, train_op, _, train_distance_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) test_loss_op, test_accuracy_op, test_xentropy_op = model.build_test_op( ) # Data load in memeory ############################################################################### print("start to load data set.") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) train_batch_patcher.bulk_load_in_memory( sess, t_ids, t_images, t_labels) validation_batch_patcher.bulk_load_in_memory( sess, v_ids, v_images, v_labels) start_time = time.time() # Model Initialization ########################################################################### # init params: we share the initial epochs. See paper. if pretrain != 0: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str( lr_values[0]) + "_e=" + str(pretrain) + "/" minus_start_time = 0 with open(file_dir + "text_log.csv") as f: for line in f: print(line, end="") text_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") print("shared weight is successfully loaded") else: sess.run(tf.global_variables_initializer()) # Traning for epoch in range(pretrain, training_epochs): if epoch < warm_up_period: is_warm_up = True else: is_warm_up = False ###################### # after 1 epoch, update loss table and calculate importance based on the paper # "https://arxiv.org/pdf/1511.06343.pdf", Title: Online Batch Selection For Faster Training of Neural Networks###################################### loss_map = {} for i in range( train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch( i) xentropy = sess.run( test_xentropy_op, feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) for j in range(len(ids)): sample_id = ids[j] if sample_id in loss_map: continue else: loss_map[sample_id] = xentropy[j] if len(loss_map) != input_reader.num_train_images: print( "error: size of loss map is not equal to the size of data set." ) train_batch_patcher.update_importance_table_based_on_online_bs( loss_map) ########################################################################################################################################################### avg_mini_loss = 0.0 avg_mini_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): # Next batch depends on the method: online batch ids, images, labels = train_batch_patcher.get_next_mini_batch( num_of_sample=batch_size, is_warm_up=is_warm_up) # Parameter updates mini_loss, mini_acc, _, distance = sess.run( [ train_loss_op, train_accuracy_op, train_op, train_distance_op ], feed_dict={ model.train_image_placeholder: images, model.train_label_placeholder: labels }) avg_mini_loss += mini_loss avg_mini_acc += mini_acc avg_mini_loss /= train_batch_patcher.num_iters_per_epoch avg_mini_acc /= train_batch_patcher.num_iters_per_epoch # (2) Compute training loss and error avg_train_loss = 0.0 avg_train_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch( i) train_loss, train_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_train_loss += train_loss avg_train_acc += train_acc avg_train_loss /= train_batch_patcher.num_iters_per_epoch avg_train_acc /= train_batch_patcher.num_iters_per_epoch # (3) Validation (or test) loss and error avg_val_loss = 0.0 avg_val_acc = 0.0 for i in range( validation_batch_patcher.num_iters_per_epoch): ids, images, labels = validation_batch_patcher.get_init_mini_batch( i) val_loss, val_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_val_loss += val_loss avg_val_acc += val_acc avg_val_loss /= validation_batch_patcher.num_iters_per_epoch avg_val_acc /= validation_batch_patcher.num_iters_per_epoch # Log Writing #################################################################################### cur_lr = sess.run(model.learning_rate) print((epoch + 1), ", ", int(time.time() - start_time), ", ", cur_lr, ", ", avg_mini_loss, ", ", (1.0 - avg_mini_acc), ", ", avg_train_loss, ", ", (1.0 - avg_train_acc), ", ", avg_val_loss, ", ", (1.0 - avg_val_acc)) text_log.append( str(epoch + 1) + ", " + str(int(time.time() - start_time)) + ", " + str(cur_lr) + ", " + str(avg_mini_loss) + ", " + str(1.0 - avg_mini_acc) + ", " + str(avg_train_loss) + ", " + str(1.0 - avg_train_acc) + ", " + str(avg_val_loss) + ", " + str(1.0 - avg_val_acc)) coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/text_log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def RandomBatch(gpu_id, input_reader, model_type, training_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, pretrain=0, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) text_log = [] text_log.append("epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error") num_train_images = input_reader.num_train_images num_val_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.width, input_reader.height, input_reader.depth] train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) validation_batch_patcher = patcher.BatchPatcher(num_val_images, batch_size, num_label) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: # Input Graph Generation ############################################################################# t_ids, t_images, t_labels = input_reader.data_read(batch_size, train=True) v_ids, v_images, v_labels = input_reader.data_read(batch_size, train=False) # Model Graph Construction ########################################################################### if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label, batch_size, batch_size) elif model_type == "WideResNet16-8": model = WideResNet(16, 8, image_shape, num_label, batch_size, batch_size) train_loss_op, train_accuracy_op, train_op, _, train_distance_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) test_loss_op, test_accuracy_op, _ = model.build_test_op() # Data load in memeory ############################################################################### print("start to load data set.") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) train_batch_patcher.bulk_load_in_memory(sess, t_ids, t_images, t_labels) validation_batch_patcher.bulk_load_in_memory(sess, v_ids, v_images, v_labels) start_time = time.time() # Model Initialization ########################################################################### # init params: we share the initial epochs. See paper. if pretrain != 0: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str(lr_values[0]) + "_e=" + str(pretrain) + "/" minus_start_time = 0 with open(file_dir + "text_log.csv") as f: for line in f: print(line, end="") text_log.append(line) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") print("shared weight is successfully loaded") else: sess.run(tf.global_variables_initializer()) # Traing Process ##################################################################################### for epoch in range(pretrain, training_epochs): # (1) Mini-batch loss and error along with netowrk updates avg_mini_loss = 0.0 avg_mini_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): # if is_warm_up = True, then select next batch samples uniformly at random ids, images, labels = train_batch_patcher.get_next_mini_batch(num_of_sample=batch_size, is_warm_up=True) mini_loss, mini_acc, _, distance = sess.run([train_loss_op, train_accuracy_op, train_op, train_distance_op], feed_dict={model.train_image_placeholder: images, model.train_label_placeholder: labels}) avg_mini_loss += mini_loss avg_mini_acc += mini_acc avg_mini_loss /= train_batch_patcher.num_iters_per_epoch avg_mini_acc /= train_batch_patcher.num_iters_per_epoch # (2) Compute training loss and error avg_train_loss = 0.0 avg_train_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch(i) train_loss, train_acc = sess.run([test_loss_op, test_accuracy_op], feed_dict={model.test_image_placeholder: images, model.test_label_placeholder: labels}) avg_train_loss += train_loss avg_train_acc += train_acc avg_train_loss /= train_batch_patcher.num_iters_per_epoch avg_train_acc /= train_batch_patcher.num_iters_per_epoch # (3) Validation (or test) loss and error avg_val_loss = 0.0 avg_val_acc = 0.0 for i in range(validation_batch_patcher.num_iters_per_epoch): ids, images, labels = validation_batch_patcher.get_init_mini_batch(i) val_loss, val_acc = sess.run([test_loss_op, test_accuracy_op], feed_dict={model.test_image_placeholder: images, model.test_label_placeholder: labels}) avg_val_loss += val_loss avg_val_acc += val_acc avg_val_loss /= validation_batch_patcher.num_iters_per_epoch avg_val_acc /= validation_batch_patcher.num_iters_per_epoch # Log Writing #################################################################################### cur_lr = sess.run(model.learning_rate) print((epoch + 1), ", ", int(time.time() - start_time) ,", ", cur_lr, ", ", avg_mini_loss, ", ", (1.0-avg_mini_acc), ", ", avg_train_loss, ", ", (1.0-avg_train_acc), ", ", avg_val_loss, ", ", (1.0-avg_val_acc)) text_log.append(str(epoch + 1) + ", " + str(int(time.time() - start_time)) + ", " + str(cur_lr) + ", " + str(avg_mini_loss) + ", " + str(1.0-avg_mini_acc) + ", " + str(avg_train_loss) + ", " + str(1.0-avg_train_acc) + ", " + str(avg_val_loss) + ", " + str(1.0-avg_val_acc)) coord.request_stop() coord.join(threads) sess.close() # Log Flushing f = open(log_dir + "/text_log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def coteaching(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, T_k=15, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) # log list text_log = [] text_log.append("epcoh, learning rate, training loss (network1), training error (network1), training loss (network2), training error (network2), " "test loss (network1), test error (network1), test loss (network2), test error (network2)\n") num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False) if model_type == "DenseNet-25-12": model1 = DenseNet(25, 12, image_shape, num_label, scope='network1') model2 = DenseNet(25, 12, image_shape, num_label, scope='network2') elif model_type == "DenseNet-40-12": model1 = DenseNet(40, 12, image_shape, num_label, scope='network1') model2 = DenseNet(40, 12, image_shape, num_label, scope='network2') elif model_type == "DenseNet-10-12": model1 = DenseNet(10, 12, image_shape, num_label, scope='network1') model2 = DenseNet(10, 12, image_shape, num_label, scope='network2') elif model_type == "VGG-19": model1 = VGG19(image_shape, num_label, scope='network1') model2 = VGG19(image_shape, num_label, scope='network2') # register training operations on Trainer class trainer1 = Trainer(model1) trainer1.train_loss_op, trainer1.train_accuracy_op, trainer1.train_op, trainer1.train_xentropy_op, trainer1.train_prob_op = model1.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer1.test_loss_op, trainer1.test_accuracy_op, trainer1.test_xentropy_op, trainer1.test_prob_op = model1.build_test_op() trainer2 = Trainer(model2) trainer2.train_loss_op, trainer2.train_accuracy_op, trainer2.train_op, trainer2.train_xentropy_op, trainer2.train_prob_op = model2.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer2.test_loss_op, trainer2.test_accuracy_op, trainer2.test_xentropy_op, trainer2.test_prob_op = model2.build_test_op() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # give noise on data set train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### sess.run(tf.global_variables_initializer()) # for model 1 for epoch in range(total_epochs): ratio = 1.0 - noise_rate * np.fmin(1.0, float(float(epoch) + 1.0) / float(T_k)) # print("Clean sample ratio: ", ratio) for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_next_random_mini_batch(batch_size) # losses of model 1 xentropy_array1 = sess.run(trainer1.train_xentropy_op, feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels}) # losses of model 2 xentropy_array2 = sess.run(trainer2.train_xentropy_op, feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels}) # choose R(T)% small-loss instances ids_for_2, images_for_2, labels_for_2 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array1, ratio) ids_for_1, images_for_1, labels_for_1 = get_lower_loss_instances(batch_size, ids, images, labels, xentropy_array2, ratio) # model update _, _ = sess.run([trainer1.train_op, trainer2.train_op], feed_dict={trainer1.model.train_image_placeholder: images_for_1, trainer1.model.train_label_placeholder: labels_for_1, trainer2.model.train_image_placeholder: images_for_2, trainer2.model.train_label_placeholder: labels_for_2}) # inference and test error for logs # test avg_val_loss_1 = 0.0 avg_val_acc_1 = 0.0 avg_val_loss_2 = 0.0 avg_val_acc_2 = 0.0 for i in range(test_batch_patcher.num_iters_per_epoch): ids, images, labels = test_batch_patcher.get_init_mini_batch(i) val_loss_1, val_acc_1 = sess.run([trainer1.test_loss_op, trainer1.test_accuracy_op], feed_dict={trainer1.model.test_image_placeholder: images, trainer1.model.test_label_placeholder: labels}) val_loss_2, val_acc_2 = sess.run([trainer2.test_loss_op, trainer2.test_accuracy_op], feed_dict={trainer2.model.test_image_placeholder: images, trainer2.model.test_label_placeholder: labels}) avg_val_loss_1 += val_loss_1 avg_val_acc_1 += val_acc_1 avg_val_loss_2 += val_loss_2 avg_val_acc_2 += val_acc_2 avg_val_loss_1 /= test_batch_patcher.num_iters_per_epoch avg_val_acc_1 /= test_batch_patcher.num_iters_per_epoch avg_val_loss_2 /= test_batch_patcher.num_iters_per_epoch avg_val_acc_2 /= test_batch_patcher.num_iters_per_epoch # Inference avg_train_loss_1 = 0.0 avg_train_acc_1 = 0.0 avg_train_loss_2 = 0.0 avg_train_acc_2 = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch(i) imp_loss_1, imp_acc_1 = sess.run([trainer1.train_loss_op, trainer1.train_accuracy_op], feed_dict={trainer1.model.train_image_placeholder: images, trainer1.model.train_label_placeholder: labels}) imp_loss_2, imp_acc_2 = sess.run([trainer2.train_loss_op, trainer2.train_accuracy_op], feed_dict={trainer2.model.train_image_placeholder: images, trainer2.model.train_label_placeholder: labels}) avg_train_loss_1 += imp_loss_1 avg_train_acc_1 += imp_acc_1 avg_train_loss_2 += imp_loss_2 avg_train_acc_2 += imp_acc_2 avg_train_loss_1 /= train_batch_patcher.num_iters_per_epoch avg_train_acc_1 /= train_batch_patcher.num_iters_per_epoch avg_train_loss_2 /= train_batch_patcher.num_iters_per_epoch avg_train_acc_2 /= train_batch_patcher.num_iters_per_epoch cur_lr = sess.run(trainer1.model.learning_rate) print((epoch + 1), ", ", cur_lr, ", ", avg_train_loss_1, ", ", avg_train_acc_1, ", ", avg_train_loss_2, ", ", avg_train_acc_2, ", ", avg_val_loss_1, ", ", avg_val_acc_1, ", ", avg_val_loss_2, ", ", avg_val_acc_2) text_log.append(str(epoch + 1) + ", " + str(cur_lr) + ", " + str(avg_train_loss_1) + ", " + str(1.0 - avg_train_acc_1) + ", " + str(avg_train_loss_2) + ", " + str(1.0 - avg_train_acc_2) + ", " + str(avg_val_loss_1) + ", " + str(1.0 - avg_val_acc_1) + ", " + str(avg_val_loss_2) + ", " + str(1.0 - avg_val_acc_2)) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def selfie(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, threshold=0.05, queue_size=15, restart=3, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "selfie" # log list training_log = [] training_log.append("epcoh, learning rate, training loss, training error, test loss, test error\n") correction_log = [] num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # corrector to classify clean and refurbishable samples correcter = self_correcter.Correcter(num_train_images, num_label, queue_size, threshold, loaded_data=train_batch_patcher.loaded_data) config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config = config) as sess: train_ids, train_images, train_labels = input_reader.data_read(batch_size, train = True) test_ids, test_images, test_labels = input_reader.data_read(batch_size, train = False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op(lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op() trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord = coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory(sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # clean data set => simulated noisy data set train_batch_patcher.set_noise(noise_rate, noise_type) ######################## main methodology for training ####################### # exception handler if queue_size > warm_up: sys.exit(-1) print("warm-up parameter error!") # start training train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=True), correction_log) for r in range(restart+1): print("# run: ", (r+1)) sess.run(trainer.init_op) # warm-up periods training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", correcter=correcter, training_log=training_log) # selfie training(sess, total_epochs-warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, noise_rate=noise_rate, correcter=correcter, training_log=training_log, correction_log=correction_log) # corrected confusion matrix train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix(entire=False), correction_log) # clean prediction history for next run correcter.predictions_clear() # recalculate noise rate of data based on refurbished samples noise_rate = np.fmin(noise_rate, correcter.compute_new_noise_ratio()) print("new noise rate: ", noise_rate) correction_log.append("new noise rate, " + str(noise_rate)) ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close() f = open(log_dir + "/detail_log.csv", "w") for text in correction_log: f.write(text + "\n") f.close()
def selfie(gpu_id, input_reader, model_type, total_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, noise_rate, noise_type, warm_up, threshold, queue_size, restart=2, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) method = "selfie" # log list training_log = [] num_train_images = input_reader.num_train_images num_test_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.height, input_reader.width, input_reader.depth] # batch pathcer train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label) test_batch_patcher = patcher.BatchPatcher(num_test_images, batch_size, num_label) # online self label correcter correcter = self_correcter.Correcter( num_train_images, num_label, queue_size, threshold, loaded_data=train_batch_patcher.loaded_data) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: train_ids, train_images, train_labels = input_reader.data_read( batch_size, train=True) test_ids, test_images, test_labels = input_reader.data_read( batch_size, train=False) if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label) elif model_type == "DenseNet-40-12": model = DenseNet(40, 12, image_shape, num_label) elif model_type == "DenseNet-10-12": model = DenseNet(10, 12, image_shape, num_label) elif model_type == "VGG-19": model = VGG19(image_shape, num_label) # register training operations on Trainer class trainer = Trainer(model) trainer.train_loss_op, trainer.train_accuracy_op, trainer.train_op, trainer.train_xentropy_op, trainer.train_prob_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) trainer.test_loss_op, trainer.test_accuracy_op, trainer.test_xentropy_op, trainer.test_prob_op = model.build_test_op( ) trainer.init_op = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # load data set in main memory train_batch_patcher.bulk_load_in_memory( sess, train_ids, train_images, train_labels) test_batch_patcher.bulk_load_in_memory(sess, test_ids, test_images, test_labels) # noise injection train_batch_patcher.set_noise(noise_rate, noise_type) #train_batch_patcher.print_transition_matrix(train_batch_patcher.get_current_noise_matrix()) ######################## main methodology for training ####################### # 1. classify clean samples using prediction certainty print("history length : ", queue_size) if queue_size > warm_up: sys.exit(-1) print("warm-up parameter error!") for i in range(restart + 1): print("Restart: ", i) sess.run(trainer.init_op) # warm-up training(sess, warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, 0, method="warm-up", correcter=correcter, training_log=training_log) # self online correction mechanism training(sess, total_epochs - warm_up, batch_size, train_batch_patcher, test_batch_patcher, trainer, warm_up, method=method, noise_rate=noise_rate, correcter=correcter, training_log=training_log) correcter.predictions_clear() ############################################################################## coord.request_stop() coord.join(threads) sess.close() f = open(log_dir + "/log.csv", "w") for text in training_log: f.write(text + "\n") f.close()