def task_e(): models = [] for i in range(3): config = load_config("e{}".format(i + 1)) model = Neuralnetwork(config) x_train, y_train = load_data(path="./", mode="train") x_test, y_test = load_data(path="./", mode="t10k") train_size = 0.9 size = len(x_train) x_valid, y_valid = x_train[int(size * train_size):], y_train[int(size * train_size):] x_train, y_train = x_train[:int(size * train_size)], y_train[:int(size * train_size)] train(model, x_train, y_train, x_valid, y_valid, config) models.append(model) test_acc = test(model, x_test, y_test) print("Test accuracy: {}".format(test_acc)) for model in models: plot_loss(model, "Loss for different activation functions", False) plt.legend([ "Training loss - Tanh", "Validation loss - Tanh", "Training loss - Sigmoid", "Validation loss - Sigmoid", "Training loss - ReLU", "Validation loss - ReLU" ]) plt.show() for model in models: plot_acc(model, "Accuracy for different activation functions", False) plt.legend([ "Training accuracy - Tanh", "Validation accuracy - Tanh", "Training accuracy - Sigmoid", "Validation accuracy - Sigmoid", "Training accuracy - ReLU", "Validation accuracy - ReLU" ]) plt.show()
def step2(source, target, epoch, batch_size=64, g_lr=0.0001, d_lr=0.0001, source_dir='./Log/ADDA/source_network/best/MNIST/NOBN', logdir='./Log/ADDA/advermodel/best/MNIST2USPS/NOBN', classes_num=10, strn=None, sten=None, ttrn=None, tten=None): # prepare data data_func = dataset.get_dataset(source, target) print(data_func) s_x_tr, s_y_tr, s_x_te, s_y_te, s_tr_size, s_te_size, s_init = data_func[ 0](batch_size, strn, sten) t_x_tr, t_y_tr, t_x_te, t_y_te, t_tr_size, t_te_size, t_init = data_func[ 1](batch_size, ttrn, tten) print( "dataset information:\n source: %s train_size: %d, test_size: %d \n target: %s train_size: %d, test_size: %d" % (source, s_tr_size, s_te_size, target, t_tr_size, t_te_size)) # create graph nn = adda.ADDA(classes_num) # for source domain feat_s = nn.s_encoder(s_x_tr, reuse=False, trainable=False) logits_s = nn.classifier(feat_s, reuse=False, trainable=False) disc_s = nn.discriminator(feat_s, reuse=False) # for target domain feat_t = nn.t_encoder(t_x_tr, reuse=False) logits_t = nn.classifier(feat_t, reuse=True, trainable=False) disc_t = nn.discriminator(feat_t, reuse=True) # build inference for test accuracy feats_s_te = nn.s_encoder(s_x_te, reuse=True, trainable=False) logits_s_te = nn.classifier(feats_s_te, reuse=True, trainable=False) disc_s_te = nn.discriminator(feats_s_te, reuse=True, trainable=False) feats_t_te = nn.t_encoder(t_x_te, reuse=True, trainable=False) logits_t_te = nn.classifier(feats_t_te, reuse=True, trainable=False) disc_t_te = nn.discriminator(feats_t_te, reuse=True, trainable=False) # build loss g_loss, d_loss = nn.build_ad_loss(disc_s, disc_t) #g_loss,d_loss = nn.build_w_loss(disc_s,disc_t) # create optimizer for two task var_t_en = tf.trainable_variables(nn.t_e) optim_g = tf.train.AdamOptimizer(g_lr, beta1=0.5, beta2=0.999).minimize(g_loss, var_list=var_t_en) #optim_g = tf.train.RMSPropOptimizer(learning_rate=0.0001).minimize(g_loss,var_list=var_t_en) var_d = tf.trainable_variables(nn.d) optim_d = tf.train.AdamOptimizer(d_lr, beta1=0.5, beta2=0.999).minimize(d_loss, var_list=var_d) #optim_d = tf.train.RMSPropOptimizer(learning_rate=0.0001).minimize(d_loss,var_list=var_d) #clip_D = [var.assign(tf.clip_by_value(var,-0.01,0.01)) for var in var_d] # create acuuracy op with training batch acc_tr_s = nn.eval(logits_s, s_y_tr) acc_tr_t = nn.eval(logits_t, t_y_tr) acc_te_s = nn.eval(logits_s_te, s_y_te) acc_te_t = nn.eval(logits_t_te, t_y_te) # create source saver for restore s_encoder encoder_path = tf.train.latest_checkpoint(source_dir + "/encoder") classifier_path = tf.train.latest_checkpoint(source_dir + "/classifier") if encoder_path is None: raise ValueError("Don't exits in this dir") if classifier_path is None: raise ValueError("Don't exits in this dir") source_var = tf.contrib.framework.list_variables(encoder_path) var_s_g = tf.global_variables(scope=nn.s_e) var_c_g = tf.global_variables(scope=nn.c) var_t_g = tf.trainable_variables(scope=nn.t_e) # print("+++++++++++++++") # print("s_encoder:",len(var_s_g)) # print(var_s_g) # print("t_encoder:",len(var_t_g)) # print(var_t_g) # print("source s_encoder:",len(source_var)) # print(source_var) # print("+++++++++++++++") encoder_saver = tf.train.Saver(var_list=var_s_g) classifier_saver = tf.train.Saver(var_list=var_c_g) dict_var = {} #print(type(source_var[0][0])) #print(type(var_t_g[0].name)) for i in source_var: for j in var_t_g: if i[0][1:] in j.name[1:]: dict_var[i[0]] = j #print(dict_var) fine_turn_saver = tf.train.Saver(var_list=dict_var) #assert False # create this model saver utils.fresh_dir(logdir) best_saver = tf.train.Saver(max_to_keep=3) # create a list to record accuracy eval_acc = [] best_acc = 0 merge = tf.summary.merge_all() # start a session config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # init t_e and d sess.run(tf.global_variables_initializer()) # init s_e and c encoder_saver.restore(sess, encoder_path) classifier_saver.restore(sess, classifier_path) fine_turn_saver.restore(sess, encoder_path) print("model init successfully!") filewriter = tf.summary.FileWriter(logdir=logdir, graph=sess.graph) for i in range(epoch): _, d_loss_, = sess.run([optim_d, d_loss]) _, g_loss_, merge_ = sess.run([optim_g, g_loss, merge]) filewriter.add_summary(merge_, global_step=i) if i % 20 == 0: print("step:{},g_loss:{:.4f},d_loss:{:.4f}".format( i, g_loss_, d_loss_)) if i % 100 == 0 or i > (epoch - 100): sess.run([s_init, t_init]) s_acc, t_acc, sx, sfe, sl, tx, tfe, tl = sess.run([ acc_te_s, acc_te_t, s_x_te, logits_s_te, s_y_te, t_x_te, logits_t_te, t_y_te ]) eval_acc.append(t_acc) if best_acc < t_acc: best_acc = t_acc print( "epoch: %d, source accuracy: %.4f, target accuracy: %.4f, best accuracy:%4f" % (i, s_acc, t_acc, best_acc)) best_saver.save(sess, logdir + "/adda_model.ckpt") utils.plot_acc(eval_acc, threshold=0.766) plt.show()
def step1(source="MNIST", batch_size=64, epoch=10, lr=0.001, logdir="./Log/ADDA/source_network/best/MNIST/NOBN", training_size=None, testing_size=None, classes_num=10): data_func = dataset.get_dataset_v2(source) x_tr, y_tr, x_te, y_te, tr_size, te_size, te_init = data_func( batch_size, training_size, testing_size) print("Training size:{},Testing size:{}".format(tr_size, te_size)) batch_num = int(tr_size / batch_size) nn = adda.ADDA(classes_num) # inference classification network fc1 = nn.s_encoder(x_tr) logits = nn.classifier(fc1) # build loss and create optimizer c_loss = nn.build_classify_loss(logits, y_tr) train_op = tf.train.AdamOptimizer(lr).minimize(c_loss) # build training accuracy with training batch tr_acc = nn.eval(logits, y_tr) # build testing accuracy with testing data logits_te = nn.classifier(nn.s_encoder(x_te, reuse=True), reuse=True) te_acc = nn.eval(logits_te, y_te) # build saver to save best epoch var_s_en = tf.trainable_variables(scope=nn.s_e) var_c = tf.trainable_variables(scope=nn.c) encoder_saver = tf.train.Saver(max_to_keep=3, var_list=var_s_en) classifier_saver = tf.train.Saver(max_to_keep=3, var_list=var_c) # keep the logdir is empty utils.fresh_dir(logdir) # create a list to record accuracy in every batch eval_acc = [] best_acc = 0 # start a session config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) for i in range(epoch): for j in range(batch_num): _, loss, tr_acc_ = sess.run([train_op, c_loss, tr_acc]) if j % 500 == 0: print("epoch:{},batch_id:{},loss:{:.4f},tr_acc:{:.4f}". format(i, j, loss, tr_acc_)) sess.run(te_init) te_acc_ = sess.run(te_acc) eval_acc.append(te_acc_) if best_acc < te_acc_: best_acc = te_acc_ encoder_saver.save(sess, logdir + "/encoder/encoder.ckpt") classifier_saver.save(sess, logdir + "/classifier/classifier.ckpt") print("#+++++++++++++++++++++++++++++++++++#") print("epoch:{},test_accuracy:{:.4f},best_acc:{:.4f}".format( i, te_acc_, best_acc)) print("#+++++++++++++++++++++++++++++++++++#") utils.plot_acc(eval_acc, threshold=0.97, name=source + " test accuracy") plt.show()
def train_using_pretrained_model(images, labels, path, net, epochs=10, learning_rate=0.0001, batch_size=32): best_accuracy = 0.0 train_loss, test_loss = [], [] train_acc, test_acc = [], [] roc = [] roc_score = [] roc_true = [] criterion = nn.BCELoss() optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate) # Training data X_train, X_test, y_train, y_test = train_test_split(images, labels, test_size=0.2, random_state=6) train_data = Dataset(X_train, y_train) train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size) # Testing data test_data = Dataset(X_test, y_test) test_loader = torch.utils.data.DataLoader(test_data, shuffle=True, batch_size=batch_size) for epoch in range(epochs): train_loss_it, train_acc_it = [], [] test_loss_it, test_acc_it = [], [] roc_score_it, roc_true_it = [], [] total_step = len(train_loader) for i, (images, labels) in enumerate(train_loader): images = images.reshape(len(images), 1, 224, 224) labels = labels optimizer.zero_grad() outputs = net(images) loss = criterion(outputs.double(), labels) loss.backward() optimizer.step() # Accuracy predicted = torch.round(outputs.data) total = labels.size(0) * labels.size(1) correct = (predicted == labels).sum().item() accuracy = 100 * correct / total print('Epoch [{}/{}], Step [{}/{}], Train-Loss: {:.4f}, Train-Acc: {:.2f} %' .format(epoch + 1, epochs, i + 1, total_step, loss.item(), accuracy)) train_acc_it.append(accuracy) train_loss_it.append(loss.item()) train_acc.append(np.mean(np.array(train_acc_it))) train_loss.append(np.mean(np.array(train_loss_it))) total = 0.0 correct = 0.0 total_step = len(test_loader) for i, (images, labels) in enumerate(test_loader): images = images.reshape(len(images), 1, 224, 224) labels = labels outputs = net(images) loss = criterion(outputs.double(), labels) predicted = torch.round(outputs.data) total += labels.size(0) * labels.size(1) correct += (predicted == labels).sum().item() accuracy = 100 * correct / total true = np.array(labels).reshape(-1) score = np.array(outputs.data).reshape(-1) roc.append(roc_auc_score(true, score)) roc_score_it.extend(np.array(outputs.data).reshape(-1)) roc_true_it.extend(np.array(labels).reshape(-1)) test_acc_it.append(accuracy) test_loss_it.append(loss.item()) test_accuracy = 100 * correct / total test_acc.append(np.mean(np.array(test_acc_it))) test_loss.append(np.mean(np.array(test_loss_it))) print('[Test] Epoch [{}/{}], Acc: {:.2f}'.format(epoch + 1, epochs, test_accuracy)) if test_accuracy > best_accuracy: torch.save(net.state_dict(), path) best_accuracy = test_accuracy if (epoch + 1) % 10 == 0: roc_score.append(roc_score_it) roc_true.append(roc_true_it) # ROC if epochs > 9: true = np.array(roc_true) score = np.array(roc_score) plot_roc_binary(true, score, './results/transfer_bin_roc.pdf', 'Transfer Binary Classifier COVID') plot_roc(roc, './results/transfer_bin_roc_auc.pdf', 'Transfer Binary Classifier COVID') plot_loss(train_loss, test_loss, './results/transfer_bin_loss.pdf', 'Transfer Binary Classifier COVID') plot_acc(train_acc, test_acc, './results/transfer_bin_acc.pdf', 'Transfer Binary Classifier COVID')
def train_model(images, labels, path, epochs=10, learning_rate=0.0001, batch_size=32): net = Net() train_loss, test_loss, = [], [] train_acc, test_acc, = [], [] roc_score = [] roc_true = [] roc=[] # Loss and optimizer criterion = nn.BCELoss() optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate) # Generate dataset X_train, X_test, y_train, y_test = train_test_split(images, labels, test_size=0.2, random_state=6) train_data = Dataset(X_train, y_train) train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size) test_data = Dataset(X_test, y_test) test_loader = torch.utils.data.DataLoader(test_data, shuffle=True, batch_size=batch_size) for epoch in range(epochs): train_loss_it, train_acc_it = [], [] test_loss_it, test_acc_it = [], [] roc_score_it, roc_true_it = [], [] net.train() total_step = len(train_loader) for i, (images, labels) in enumerate(train_loader): # Move tensors to the configured device images = images.reshape(len(images), 1, 224, 224) labels = labels # Forward pass outputs = net(images) loss = criterion(outputs, labels) # Accuracy predicted = torch.round(outputs.data).reshape(len(labels)) total = labels.size(0) correct = (predicted == labels).sum().item() accuracy = 100 * correct / total # Backward and optimize optimizer.zero_grad() loss.backward() optimizer.step() print('[Train] Epoch [{}/{}], Step [{}/{}], Train-Loss: {:.4f}, Train-Acc: {:.2f} %' .format(epoch + 1, epochs, i + 1, total_step, loss.item(), accuracy)) train_acc_it.append(accuracy) train_loss_it.append(loss.item()) train_acc.append(np.mean(np.array(train_acc_it))) train_loss.append(np.mean(np.array(train_loss_it))) net.eval() with torch.no_grad(): correct = 0 total = 0 total_step = len(test_loader) for i, (images, labels) in enumerate(test_loader): images = images.reshape(len(images), 1, 224, 224) labels = labels outputs = net(images) loss = criterion(outputs, labels) predicted = torch.round(outputs.data).reshape(len(labels)) total += labels.size(0) correct += (predicted == labels).sum().item() accuracy = 100 * correct / total roc_score_it.extend(np.array(outputs.data).reshape(-1)) roc_true_it.extend(np.array(labels).reshape(-1)) test_acc_it.append(accuracy) test_loss_it.append(loss.item()) true = np.array(labels).reshape(-1) score = np.array(outputs.data).reshape(-1) roc.append(roc_auc_score(true, score)) print('[Test] Epoch [{}/{}], Step [{}/{}], Test-Loss: {:.4f}, Test-Acc: {:.2f}%' .format(epoch + 1, epochs, i + 1, total_step, loss.item(), accuracy)) if (epoch + 1) % 10 == 0: roc_score.append(roc_score_it) roc_true.append(roc_true_it) test_acc.append(np.mean(np.array(test_acc_it))) test_loss.append(np.mean(np.array(test_loss_it))) # Save the model checkpoint torch.save(net.state_dict(), path) # ROC if epochs > 9: true = np.array(roc_true) score = np.array(roc_score) plot_roc_binary(true, score, './results/bin_roc.pdf', 'Binary Classifier COVID') plot_loss(train_loss, test_loss, './results/bin_loss.pdf', 'Binary Classifier COVID') plot_acc(train_acc, test_acc, './results/bin_acc.pdf', 'Binary Classifier COVID') plot_roc(roc, './results/bin_roc_auc.pdf', 'Simple Binary Classifier COVID') return net
def train(istrain=True, model_type='quadmodal_1', saved_model_path=None, task='emotion', batch_size=2, nb_epoch=200, learning_r=1e-3, show_plots=True, is_fusion=False, fusion_type=None, pretrained=False): """ train the model :param model: 'visual_model','audio_model','word_model','trimodal_model','quadmodal_X_model' :param saved_model_path: saved_model path :param task: 'aoursal','valence','emotion' :param batch_size: 2 :param nb_epoch:2100 :return:s """ timestamp = time.strftime('%Y-%m-%d-%H:%M:%S',time.localtime(time.time())) # Helper: Save the model. model_name = model_type model_name = model_name.replace(':','-') model_name = model_name.replace('[','') model_name = model_name.replace(']','') if ',' in model_name: model_name = model_name.replace(',','__') max_len = 200 if len(model_name) >= max_len: model_name = model_name[:max_len] model_name = 'fusion_' + fusion_type + '__' + model_name if not os.path.exists(os.path.join('checkpoints', model_name)): os.makedirs(os.path.join('checkpoints', model_name)) checkpointer = ModelCheckpoint( monitor='val_acc', #filepath = os.path.join('checkpoints', model, task+'-'+ str(timestamp)+'-'+'best.hdf5' ), filepath = os.path.join('checkpoints', model_name, task + '-{val_acc:.3f}-{acc:.3f}.hdf5' ), verbose=1, save_best_only=True) checkpointer_acc = ModelCheckpoint( monitor='acc', #filepath = os.path.join('checkpoints', model, task+'-'+ str(timestamp)+'-'+'best.hdf5' ), filepath = os.path.join('checkpoints', model_name, task + '-{val_acc:.3f}-{acc:.3f}.hdf5' ), verbose=1, save_best_only=True) # Helper: TensorBoard tb = TensorBoard(log_dir=os.path.join('logs', model_name)) # Helper: Stop when we stop learning. early_stopper = EarlyStopping(patience=1000) # Helper: Save results. csv_logger = CSVLogger(os.path.join('logs', model_name , task +'-'+ \ str(timestamp) + '.log')) # Get the data and process it. # seq_length for the sentence seq_length = 20 dataset = DataSet( istrain = istrain, model = model_type, task = task, seq_length=seq_length, model_name=model_name, is_fusion=is_fusion ) # Get the model. model = None if pretrained: model_weights_path = get_best_model(model_name) if model_weights_path: print('USING MODEL', model_weights_path) model = load_model(model_weights_path) # model_file = os.path.join('models',model_name + '.hdf5') # if os.path.exists(model_file): # model = load_model(model_file) # else: # print('No trained model found') if model is None: rm = ResearchModels( istrain = istrain, model = model_type, seq_length = seq_length, saved_path=saved_model_path, task_type= task, learning_r = learning_r, model_name=model_name, is_fusion=is_fusion, fusion_type=fusion_type ) model = rm.model # Get training and validation data. x_train, y_train, train_name_list = dataset.get_all_sequences_in_memory('Train') x_valid, y_valid, valid_name_list= dataset.get_all_sequences_in_memory('Validation') x_test, y_test, test_name_list = dataset.get_all_sequences_in_memory('Test') if task == 'emotion': y_train = to_categorical(y_train) y_valid = to_categorical(y_valid) y_test = to_categorical(y_test) # Fit! # Use standard fit print('Size', len(x_train), len(y_train), len(x_valid), len(y_valid), len(x_test), len(y_test)) history = model.fit( x_train, y_train, batch_size=batch_size, validation_data=(x_valid,y_valid), verbose=1, callbacks=[tb, csv_logger, checkpointer, checkpointer_acc], #callbacks=[tb, early_stopper, csv_logger, checkpointer], #callbacks=[tb, lrate, csv_logger, checkpointer], epochs=nb_epoch) # find the current best model and get its prediction on validation set model_weights_path = get_best_model(model_name) #model_weights_path = os.path.join('checkpoints', model_name, task + '-' + str(nb_epoch) + '-' + str(timestamp) + '-' + 'best.hdf5' ) print('model_weights_path', model_weights_path) if model_weights_path: best_model = load_custom_model(model_weights_path) else: best_model = model y_valid_pred = best_model.predict(x_valid) y_valid_pred = np.squeeze(y_valid_pred) y_train_pred = best_model.predict(x_train) y_train_pred = np.squeeze(y_train_pred) y_test_pred = best_model.predict(x_test) y_test_pred = np.squeeze(y_test_pred) #calculate the ccc and mse if not os.path.exists('results'): os.mkdir('results') filename = os.path.join('results', model_name+'__'+str(nb_epoch)+'_'+task+'.txt') f1_score = f1(y_valid, y_valid_pred) f1_score_test = f1(y_test, y_test_pred) acc_val = model.evaluate(x_valid, y_valid, verbose=1)[1] acc_train = model.evaluate(x_train, y_train, verbose=1)[1] acc_test = model.evaluate(x_test, y_test, verbose=1)[1] print("F1 score in validation set is {}".format(f1_score)) print("F1 score in test set is {}".format(f1_score_test)) print("Val acc is {}".format(acc_val)) print("Train acc is {}".format(acc_train)) print("Test acc is {}".format(acc_test)) plot_acc(history, model_name, timestamp, show_plots, nb_epoch) with open(filename, 'w') as f: f.write(str([acc_val, acc_train, acc_test, f1_score, f1_score_test])) # display the prediction and true label log_path = os.path.join('logs', model_name , task +'-'+ \ str(timestamp) + '.log') display_true_vs_pred([y_valid, y_train, y_test], [y_valid_pred, y_train_pred, y_test_pred],log_path, task, model_name, [acc_val, acc_train, acc_test], show_plots, timestamp, nb_epoch)
criterion, fields=fields, metric=binary_accuracy) print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}') train_accs += [train_acc] train_losses += [train_loss] valid_accs += [valid_acc] valid_losses += [valid_loss] # scheduler.step() early_stopping(valid_loss, model) if valid_acc > 0.55 and mcc > 0.056: torch.save( { 'net': model.state_dict(), 'text': TEXT, 'dim': embed_dim, }, f'te-model-acc-{valid_acc}-{mcc}.pth') if early_stopping.early_stop: print(f"Epochs: {epoch} - Early Stopping...") break plot_acc(train_accs, valid_accs, fname=f"ternn-epochs-{epochs}-acc-{valid_acc}.png") plot_loss(train_losses, valid_losses, fname=f"ternn-epochs-{epochs}-loss-{valid_acc}.png")
valid_loss, valid_acc, mcc = evaluate(model, valid_iterator, criterion, fields=fields, metric=binary_accuracy) print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}') train_accs += [train_acc] train_losses += [train_loss] valid_accs += [valid_acc] valid_losses += [valid_loss] # scheduler.step() early_stopping(valid_loss, model) if valid_acc > 0.55 and mcc > 0.056: torch.save({ 'net': model.state_dict(), 'text': TEXT, }, f'roberta-model-acc-{valid_acc}-{mcc}.pth') if early_stopping.early_stop: print(f"Epochs: {epoch} - Early Stopping...") break plot_acc(train_accs, valid_accs, fname=f"roberta-epochs-{epochs}-acc-{valid_acc}.png") plot_loss(train_losses, valid_losses, fname=f"roberta-epochs-{epochs}-loss-{valid_acc}.png")
def acc(self): loss = [] for k in range(1, self.args.acc_k_range + 1): self._build_model(k) loss.append(float(self.predict(train=False) * 100)) plot_acc(loss)