def train_den_net(model_architecture, model_hyperpara, train_hyperpara, dataset, data_type, doLifelong, useGPU=False, GPU_device=0, save_param=False, param_folder_path='saved_param', save_param_interval=100, save_graph=False): assert (('den' in model_architecture or 'dynamically' in model_architecture) and doLifelong), "Use train function appropriate to the architecture (Dynamically Expandable Net)" print("\nTrain function for Dynamically Expandable Net is called!\n") config = tf.ConfigProto() if useGPU: os.environ["CUDA_VISIBLE_DEVICES"]=str(GPU_device) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.90 print("GPU %d is used" %(GPU_device)) else: print("CPU is used") ## This order of tasks for training can be arbitrary. task_training_order = list(range(train_hyperpara['num_tasks'])) task_for_train, task_change_epoch = task_training_order.pop(0), [1] ### set-up data train_data, validation_data, test_data = dataset if 'mnist' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = mnist_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'cifar10' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif data_type == 'cifar100': num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'officehome' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = officehome_data_print_info(train_data, validation_data, test_data, True, print_info=False) ### reformat data for DEN trainX, trainY = [train_data[t][0] for t in task_training_order], [train_data[t][1] for t in task_training_order] validX, validY = [validation_data[t][0] for t in task_training_order], [validation_data[t][1] for t in task_training_order] testX, testY = [test_data[t][0] for t in task_training_order], [test_data[t][1] for t in task_training_order] if save_graph: if 'graphs' not in os.listdir(os.getcwd()): os.mkdir(os.getcwd()+'/graphs') ### Set hyperparameter related to training process learning_step_max = train_hyperpara['learning_step_max'] improvement_threshold = train_hyperpara['improvement_threshold'] patience = train_hyperpara['patience'] patience_multiplier = train_hyperpara['patience_multiplier'] if 'batch_size' in model_hyperpara: batch_size = model_hyperpara['batch_size'] ### Generate Model learning_model, generation_success = model_generation(model_architecture, model_hyperpara, train_hyperpara, [x_dim, y_dim, y_depth, num_task]) if not generation_success: return (None, None, None, None) learning_model.set_sess_config(config) params = dict() train_accuracy, valid_accuracy, test_accuracy, best_test_accuracy = [], [], [], [] start_time = timeit.default_timer() for train_task_cnt in range(num_task): print("\n\nStart training new task %d" %(train_task_cnt)) data = (trainX, trainY, validX, validY, testX, testY) learning_model.sess = tf.Session(config=config) learning_model.T = learning_model.T + 1 learning_model.task_indices.append(train_task_cnt+1) learning_model.load_params(params, time = 1) tr_acc, v_acc, te_acc, best_te_acc = learning_model.add_task(train_task_cnt+1, data, save_param, save_graph) train_accuracy = train_accuracy+tr_acc valid_accuracy = valid_accuracy+v_acc test_accuracy = test_accuracy+te_acc best_test_accuracy = best_test_accuracy+best_te_acc params = learning_model.get_params() learning_model.destroy_graph() learning_model.sess.close() num_trainable_var = learning_model.num_trainable_var(params_list=params) end_time = timeit.default_timer() print("End of Training") print("Time consumption for training : %.2f" %(end_time-start_time)) task_change_epoch = learning_model.task_change_epoch tmp_valid_acc_hist = np.array(valid_accuracy) chk_epoch = [(task_change_epoch[x], task_change_epoch[x+1]) for x in range(len(task_change_epoch)-1)] # + [(task_change_epoch[-1], learning_step+1)] tmp_best_valid_acc_list = [np.amax(tmp_valid_acc_hist[x[0]:x[1], t]) for x, t in zip(chk_epoch, range(num_task))] ## Summary of statistics during training and evaluation result_summary = {} result_summary['training_time'] = end_time - start_time result_summary['num_epoch'] = learning_model.num_training_epoch result_summary['history_train_accuracy'] = np.array(train_accuracy) result_summary['history_validation_accuracy'] = np.array(valid_accuracy) result_summary['history_test_accuracy'] = np.array(test_accuracy) result_summary['history_best_test_accuracy'] = np.array(best_test_accuracy) result_summary['best_validation_accuracy'] = sum(tmp_best_valid_acc_list) / float(len(tmp_best_valid_acc_list)) result_summary['test_accuracy_at_best_epoch'] = 0.0 result_summary['task_changed_epoch'] = task_change_epoch[:-1] return result_summary, num_trainable_var
def train(model_architecture, model_hyperpara, train_hyperpara, dataset, data_type, doLifelong, useGPU=False, GPU_device=0, save_param=False, param_folder_path='saved_param', save_param_interval=100, save_graph=False, tfInitParam=None): assert ('progressive' not in model_architecture and 'den' not in model_architecture and 'dynamically' not in model_architecture), "Use train function appropriate to the architecture" config = tf.ConfigProto() if useGPU: os.environ["CUDA_VISIBLE_DEVICES"]=str(GPU_device) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.97 print("GPU %d is used" %(GPU_device)) else: os.environ["CUDA_VISIBLE_DEVICES"]="" print("CPU is used") ## This order of tasks for training can be arbitrary. task_training_order = list(range(train_hyperpara['num_tasks'])) task_for_train, task_change_epoch = task_training_order.pop(0), [1] ### set-up data train_data, validation_data, test_data = dataset if 'mnist' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = mnist_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif ('cifar10' in data_type) and not ('cifar100' in data_type): num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'cifar100' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'officehome' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = officehome_data_print_info(train_data, validation_data, test_data, True, print_info=False) ### Set hyperparameter related to training process learning_step_max = train_hyperpara['learning_step_max'] improvement_threshold = train_hyperpara['improvement_threshold'] patience = train_hyperpara['patience'] patience_multiplier = train_hyperpara['patience_multiplier'] batch_size = model_hyperpara['batch_size'] ### Generate Model learning_model, generation_success = model_generation(model_architecture, model_hyperpara, train_hyperpara, [x_dim, y_dim, y_depth, num_task], tfInitParam=tfInitParam) if not generation_success: return (None, None, None, None) ### Training Procedure best_param = [] if save_param: best_para_file_name = param_folder_path+'/best_model_parameter' print("Saving trained parameters at '%s'" %(param_folder_path) ) else: print("Not saving trained parameters") learning_step = -1 if (('batch_size' in locals()) or ('batch_size' in globals())) and (('num_task' in locals()) or ('num_task' in globals())): if num_task > 1: indices = [list(range(num_train[x])) for x in range(num_task)] else: indices = [list(range(num_train[0]))] best_valid_accuracy, test_accuracy_at_best_epoch, best_epoch, epoch_bias = 0.0, 0.0, -1, 0 train_accuracy_hist, valid_accuracy_hist, test_accuracy_hist, best_test_accuracy_hist = [], [], [], [] model_train_acc, model_valid_acc, model_test_acc = learning_model.train_accuracy, learning_model.valid_accuracy, learning_model.test_accuracy with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) if save_graph: tfboard_writer = tf.summary.FileWriter('./graphs', sess.graph) start_time = timeit.default_timer() while learning_step < min(learning_step_max, epoch_bias + patience): learning_step = learning_step+1 if not doLifelong: task_for_train = np.random.randint(0, num_task) #### training process if learning_step > 0: shuffle(indices[task_for_train]) for batch_cnt in range(num_train[task_for_train]//batch_size): batch_train_x = train_data[task_for_train][0][indices[task_for_train][batch_cnt*batch_size:(batch_cnt+1)*batch_size], :] batch_train_y = train_data[task_for_train][1][indices[task_for_train][batch_cnt*batch_size:(batch_cnt+1)*batch_size]] sess.run(learning_model.update[task_for_train], feed_dict={learning_model.model_input[task_for_train]: batch_train_x, learning_model.true_output[task_for_train]: batch_train_y, learning_model.epoch: learning_step-1, learning_model.dropout_prob: 0.5}) #### evaluation process train_accuracy_tmp = [0.0 for _ in range(num_task)] validation_accuracy_tmp = [0.0 for _ in range(num_task)] test_accuracy_tmp = [0.0 for _ in range(num_task)] for task_cnt in range(num_task): for batch_cnt in range(num_train[task_cnt]//batch_size): train_accuracy_tmp[task_cnt] = train_accuracy_tmp[task_cnt] + sess.run(model_train_acc[task_cnt], feed_dict={learning_model.model_input[task_cnt]: train_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output[task_cnt]: train_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) train_accuracy_tmp[task_cnt] = train_accuracy_tmp[task_cnt]/((num_train[task_cnt]//batch_size)*batch_size) for batch_cnt in range(num_valid[task_cnt]//batch_size): validation_accuracy_tmp[task_cnt] = validation_accuracy_tmp[task_cnt] + sess.run(model_valid_acc[task_cnt], feed_dict={learning_model.model_input[task_cnt]: validation_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output[task_cnt]: validation_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) validation_accuracy_tmp[task_cnt] = validation_accuracy_tmp[task_cnt]/((num_valid[task_cnt]//batch_size)*batch_size) for batch_cnt in range(num_test[task_cnt]//batch_size): test_accuracy_tmp[task_cnt] = test_accuracy_tmp[task_cnt] + sess.run(model_test_acc[task_cnt], feed_dict={learning_model.model_input[task_cnt]: test_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output[task_cnt]: test_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) test_accuracy_tmp[task_cnt] = test_accuracy_tmp[task_cnt]/((num_test[task_cnt]//batch_size)*batch_size) train_accuracy, valid_accuracy, test_accuracy = sum(train_accuracy_tmp)/num_task, sum(validation_accuracy_tmp)/num_task, sum(test_accuracy_tmp)/num_task if doLifelong: train_accuracy_to_compare, valid_accuracy_to_compare, test_accuracy_to_compare = train_accuracy_tmp[task_for_train], validation_accuracy_tmp[task_for_train], test_accuracy_tmp[task_for_train] else: train_accuracy_to_compare, valid_accuracy_to_compare, test_accuracy_to_compare = train_accuracy, valid_accuracy, test_accuracy print('epoch %d - Train : %f, Validation : %f' % (learning_step, abs(train_accuracy_to_compare), abs(valid_accuracy_to_compare))) #### when validation accuracy is better than before if valid_accuracy_to_compare > best_valid_accuracy: str_temp = '' if valid_accuracy_to_compare > best_valid_accuracy * improvement_threshold: ## for early-stopping patience = max(patience, (learning_step-epoch_bias)*patience_multiplier) str_temp = '\t<<' best_valid_accuracy, best_epoch = valid_accuracy_to_compare, learning_step test_accuracy_at_best_epoch = test_accuracy_to_compare print('\t\t\t\t\t\t\tTest : %f%s' % (abs(test_accuracy_at_best_epoch), str_temp)) #### switch to new task (only for lifelong learning) if doLifelong and learning_step >= epoch_bias+min(patience, learning_step_max//num_task) and len(task_training_order) > 0: print('\n\t>>Change to new task!<<\n') if save_param: para_file_name = param_folder_path + '/model_parameter_t%d.mat'%(task_for_train) curr_param = learning_model.get_params_val(sess) savemat(para_file_name, {'parameter': curr_param}) # update epoch_bias, task_for_train, task_change_epoch epoch_bias, task_for_train = learning_step, task_training_order.pop(0) task_change_epoch.append(learning_step+1) # initialize best_valid_accuracy, best_epoch, patience patience = train_hyperpara['patience'] best_valid_accuracy, best_epoch = 0.0, -1 train_accuracy_hist.append(train_accuracy_tmp + [train_accuracy]) valid_accuracy_hist.append(validation_accuracy_tmp + [valid_accuracy]) test_accuracy_hist.append(test_accuracy_tmp + [test_accuracy]) best_test_accuracy_hist.append(test_accuracy_at_best_epoch) if save_param: para_file_name = param_folder_path + '/final_model_parameter.mat' curr_param = learning_model.get_params_val(sess) savemat(para_file_name, {'parameter': curr_param}) end_time = timeit.default_timer() print("End of Training") print("Time consumption for training : %.2f" %(end_time-start_time)) if not doLifelong: print("Best validation accuracy : %.4f (at epoch %d)" %(abs(best_valid_accuracy), best_epoch)) print("Test accuracy at that epoch (%d) : %.4f" %(best_epoch, abs(test_accuracy_at_best_epoch))) ## Summary of statistics during training and evaluation result_summary = {} result_summary['training_time'] = end_time - start_time result_summary['num_epoch'] = learning_step result_summary['best_epoch'] = best_epoch result_summary['history_train_accuracy'] = train_accuracy_hist result_summary['history_validation_accuracy'] = valid_accuracy_hist result_summary['history_test_accuracy'] = test_accuracy_hist result_summary['history_best_test_accuracy'] = best_test_accuracy_hist result_summary['best_validation_accuracy'] = abs(best_valid_accuracy) result_summary['test_accuracy_at_best_epoch'] = abs(test_accuracy_at_best_epoch) if doLifelong: tmp_valid_accuracy_hist = np.array(valid_accuracy_hist) chk_epoch = [(task_change_epoch[x], task_change_epoch[x+1]) for x in range(len(task_change_epoch)-1)] + [(task_change_epoch[-1], learning_step+1)] tmp_best_valid_accuracy_list = [np.amax(tmp_valid_accuracy_hist[x[0]:x[1], t]) for x, t in zip(chk_epoch, range(num_task))] result_summary['best_validation_accuracy'] = sum(tmp_best_valid_accuracy_list) / float(len(tmp_best_valid_accuracy_list)) result_summary['task_changed_epoch'] = task_change_epoch if save_graph: tfboard_writer.close() return result_summary, learning_model.num_trainable_var
def train_progressive_net(model_architecture, model_hyperpara, train_hyperpara, dataset, data_type, doLifelong, useGPU=False, GPU_device=0, save_param=False, param_folder_path='saved_param', save_param_interval=100, save_graph=False): #JD's change count = 0 time_info = [] model_size = [] with open('./task_count.pickle','rb') as f: tasks_to_complete = pickle.load(f) assert ('progressive' in model_architecture and doLifelong), "Use train function appropriate to the architecture (Progressive Neural Net)" print("\nTrain function for Progressive Net is called!\n") config = tf.compat.v1.ConfigProto() if useGPU: os.environ["CUDA_VISIBLE_DEVICES"]=str(GPU_device) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.90 print("GPU %d is used" %(GPU_device)) else: print("CPU is used") ## This order of tasks for training can be arbitrary. task_training_order = list(range(train_hyperpara['num_tasks'])) task_for_train, task_change_epoch = task_training_order.pop(0), [1] ### set-up data train_data, validation_data, test_data = dataset if 'mnist' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = mnist_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'cifar10' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif data_type == 'cifar100': num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = cifar_data_print_info(train_data, validation_data, test_data, True, print_info=False) elif 'officehome' in data_type: num_task, num_train, num_valid, num_test, x_dim, y_dim, y_depth = officehome_data_print_info(train_data, validation_data, test_data, True, print_info=False) ### Set hyperparameter related to training process learning_step_max = train_hyperpara['learning_step_max'] improvement_threshold = train_hyperpara['improvement_threshold'] patience = train_hyperpara['patience'] patience_multiplier = train_hyperpara['patience_multiplier'] if 'batch_size' in model_hyperpara: batch_size = model_hyperpara['batch_size'] ### Generate Model learning_model, generation_success = model_generation(model_architecture, model_hyperpara, train_hyperpara, [x_dim, y_dim, y_depth, num_task]) if not generation_success: return (None, None, None, None) ### Training Procedure if save_param: best_para_file_name = param_folder_path+'/best_model_parameter' learning_step = -1 if (('batch_size' in locals()) or ('batch_size' in globals())) and (('num_task' in locals()) or ('num_task' in globals())): if num_task > 1: indices = [list(range(num_train[x])) for x in range(num_task)] else: indices = [list(range(num_train[0]))] best_valid_accuracy, test_accuracy_at_best_epoch, best_epoch, epoch_bias = 0.0, 0.0, -1, 0 train_accuracy_hist, valid_accuracy_hist, test_accuracy_hist, best_test_accuracy_hist = [], [], [], [] task_for_train, task_change_epoch = 0, [1] best_param = [] if not save_param: print("Not saving trained parameters") else: print("Saving trained parameters at '%s'" %(param_folder_path) ) start_time = timeit.default_timer() for train_task_cnt in range(num_task): #### Construct new task network on top of earlier task networks if train_task_cnt > 0: tf.compat.v1.reset_default_graph() learning_model.new_lifelong_task(params=param_of_prev_columns) del param_of_prev_columns model_train_accuracy, model_valid_accuracy, model_test_accuracy = learning_model.train_accuracy, learning_model.valid_accuracy, learning_model.test_accuracy with tf.compat.v1.Session(config=config) as sess: sess.run(tf.compat.v1.global_variables_initializer()) if save_graph: tfboard_writer = tf.summary.FileWriter('./graphs/prog_nn/task_' + str(train_task_cnt), sess.graph) while learning_step < min(learning_step_max, epoch_bias + patience): learning_step = learning_step+1 #### training process if learning_step > 0: shuffle(indices[task_for_train]) for batch_cnt in range(num_train[task_for_train]//batch_size): batch_train_x = train_data[task_for_train][0][indices[task_for_train][batch_cnt*batch_size:(batch_cnt+1)*batch_size], :] batch_train_y = train_data[task_for_train][1][indices[task_for_train][batch_cnt*batch_size:(batch_cnt+1)*batch_size]] sess.run(learning_model.update, feed_dict={learning_model.model_input: batch_train_x, learning_model.true_output: batch_train_y, learning_model.epoch: learning_step-1, learning_model.dropout_prob: 0.5}) #### evaluation process train_accuracy_tmp = [0.0 for _ in range(num_task)] validation_accuracy_tmp = [0.0 for _ in range(num_task)] test_accuracy_tmp = [0.0 for _ in range(num_task)] for task_cnt in range(task_for_train+1): for batch_cnt in range(num_train[task_cnt]//batch_size): train_accuracy_tmp[task_cnt] = train_accuracy_tmp[task_cnt] + sess.run(model_train_accuracy[task_cnt], feed_dict={learning_model.model_input: train_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output: train_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) train_accuracy_tmp[task_cnt] = train_accuracy_tmp[task_cnt]/((num_train[task_cnt]//batch_size)*batch_size) for batch_cnt in range(num_valid[task_cnt]//batch_size): validation_accuracy_tmp[task_cnt] = validation_accuracy_tmp[task_cnt] + sess.run(model_valid_accuracy[task_cnt], feed_dict={learning_model.model_input: validation_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output: validation_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) validation_accuracy_tmp[task_cnt] = validation_accuracy_tmp[task_cnt]/((num_valid[task_cnt]//batch_size)*batch_size) for batch_cnt in range(num_test[task_cnt]//batch_size): test_accuracy_tmp[task_cnt] = test_accuracy_tmp[task_cnt] + sess.run(model_test_accuracy[task_cnt], feed_dict={learning_model.model_input: test_data[task_cnt][0][batch_cnt*batch_size:(batch_cnt+1)*batch_size, :], learning_model.true_output: test_data[task_cnt][1][batch_cnt*batch_size:(batch_cnt+1)*batch_size], learning_model.dropout_prob: 1.0}) test_accuracy_tmp[task_cnt] = test_accuracy_tmp[task_cnt]/((num_test[task_cnt]//batch_size)*batch_size) train_accuracy, valid_accuracy, test_accuracy = sum(train_accuracy_tmp)/num_task, sum(validation_accuracy_tmp)/num_task, sum(test_accuracy_tmp)/num_task train_accuracy_to_compare, valid_accuracy_to_compare, test_accuracy_to_compare = train_accuracy_tmp[task_for_train], validation_accuracy_tmp[task_for_train], test_accuracy_tmp[task_for_train] print('epoch %d - Train : %f, Validation : %f' % (learning_step, abs(train_accuracy_to_compare), abs(valid_accuracy_to_compare))) #### when validation accuracy is better than before if valid_accuracy_to_compare > best_valid_accuracy: str_temp = '' if valid_accuracy_to_compare > best_valid_accuracy * improvement_threshold: patience = max(patience, (learning_step-epoch_bias)*patience_multiplier) str_temp = '\t<<' best_valid_accuracy, best_epoch = valid_accuracy_to_compare, learning_step test_accuracy_at_best_epoch = test_accuracy_to_compare print('\t\t\t\t\t\t\tTest : %f%s' % (test_accuracy_at_best_epoch, str_temp)) train_accuracy_hist.append(train_accuracy_tmp + [abs(train_accuracy)]) valid_accuracy_hist.append(validation_accuracy_tmp + [abs(valid_accuracy)]) test_accuracy_hist.append(test_accuracy_tmp + [abs(test_accuracy)]) best_test_accuracy_hist.append(abs(test_accuracy_at_best_epoch)) #### switch to new task (only for lifelong learning) if doLifelong and learning_step >= epoch_bias+min(patience, learning_step_max//num_task) and len(task_training_order) > 0: print('\n\t>>Change to new task!<<\n') model_size.append(get_size(learning_model)) print(model_size,'model_size') end_time = timeit.default_timer() time_info.append(end_time-start_time) print(time_info) ##### JD version for df-cnn count += 1 #print(test_accuracy_hist) if count == tasks_to_complete: with open('./tmp/res.pickle', 'wb') as f: pickle.dump(test_accuracy_hist, f) sys.exit() if save_param: para_file_name = param_folder_path + '/model_parameter_t%d.mat'%(task_for_train) curr_param = learning_model.get_params_val(sess) savemat(para_file_name, {'parameter': curr_param}) # update epoch_bias, task_for_train, task_change_epoch epoch_bias, task_for_train = learning_step, task_training_order.pop(0) task_change_epoch.append(learning_step+1) # initialize best_valid_accuracy, best_epoch, patience patience = train_hyperpara['patience'] best_valid_accuracy, best_epoch = 0.0, -1 param_of_prev_columns = learning_model.get_prev_net_param(sess=sess) break if save_graph: tfboard_writer.close() end_time = timeit.default_timer() time_info.append(end_time-start_time) print(time_info) model_size.append(get_size(learning_model)) print("End of Training") print("Time consumption for training : %.2f" %(end_time-start_time)) ###JD with open('./tmp/res.pickle', 'wb') as f: pickle.dump(test_accuracy_hist, f) with open('time_info.pickle','wb') as f: pickle.dump(time_info, f) with open('memory_info.pickle','wb') as f: pickle.dump(model_size, f) sys.exit() if not doLifelong: print("Best validation accuracy : %.4f (at epoch %d)" %(abs(best_valid_accuracy), best_epoch)) print("Test accuracy at that epoch (%d) : %.4f" %(best_epoch, abs(test_accuracy_at_best_epoch))) tmp_valid_accuracy_hist = np.array(valid_accuracy_hist) chk_epoch = [(task_change_epoch[x], task_change_epoch[x+1]) for x in range(len(task_change_epoch)-1)] + [(task_change_epoch[-1], learning_step+1)] tmp_best_valid_accuracy_list = [np.amax(tmp_valid_accuracy_hist[x[0]:x[1], t]) for x, t in zip(chk_epoch, range(num_task))] ## Summary of statistics during training and evaluation result_summary = {} result_summary['training_time'] = end_time - start_time result_summary['num_epoch'] = learning_step result_summary['best_epoch'] = best_epoch result_summary['history_train_accuracy'] = train_accuracy_hist result_summary['history_validation_accuracy'] = valid_accuracy_hist result_summary['history_test_accuracy'] = test_accuracy_hist result_summary['history_best_test_accuracy'] = best_test_accuracy_hist result_summary['best_validation_accuracy'] = sum(tmp_best_valid_accuracy_list) / float(len(tmp_best_valid_accuracy_list)) result_summary['test_accuracy_at_best_epoch'] = abs(test_accuracy_at_best_epoch) result_summary['task_changed_epoch'] = task_change_epoch return result_summary, learning_model.num_trainable_var