def train(choice, X_train, y_train, X_test, y_test): config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True student_model = None teacher_model = None with tf.Session(config=config) as sess: #Teacher Model global ssae, ae if choice['teacher'] == 'lnin': teacher_model = LNIN(img_size, img_class, sess, choice) elif choice['teacher'] == 'lnin_ae': teacher_model = DNIN(img_size, img_class, sess, choice) ae = AE(img_size, img_class, sess, choice) elif choice['teacher'] == 'lnin_ssae': teacher_model = DNIN(img_size, img_class, sess, choice) ae = AE(img_size, img_class, sess, choice) ssae = SSAE(img_size, img_class, sess, choice) elif choice['teacher'] == 'dnin': teacher_model = DNIN(img_size, img_class, sess, choice) elif choice['teacher'] == 'mnin': teacher_model = MNIN(img_size, img_class, sess, choice) elif choice['teacher'] == 'tnin': teacher_model = TNIN(img_size, img_class, sess, choice) else: teacher_model = SN(img_size, img_class, sess, choice) #Student Model if choice['student'] == 'lsn': student_model = LSN(img_size, img_class, sess, choice) elif choice['student'] == 'dsn': student_model = DSN(img_size, img_class, sess, choice) elif choice['student'] == 'msn': student_model = MSN(img_size, img_class, sess, choice) if choice['student'] != 'none': if not student_model.save: sess.run(tf.initialize_all_variables()) print "Initialize" else: if choice['teacher'] == 'lnin_ssae': if not ssae.save: sess.run(tf.initialize_all_variables()) print "Initialize" elif choice['teacher'] == 'lnin_ae': if not ae.save: sess.run(tf.initialize_all_variables()) print "Initialize" else: if not teacher_model.save: sess.run(tf.initialize_all_variables()) print "Initialize" max_epochs = choice['epochs'] l = 0 a = 0 test_accuracy = [] step = 0 epoch = 0 loss = [ ] #np.load("loss_"+choice['teacher'] +'_'+choice['student']+"_run"+str(choice['run'])+".npy").tolist() accuracy = [ ] #np.load("accuracy_"+choice['teacher']+'_'+choice['student']+"_run"+str(choice['run'])+".npy").tolist() while epoch <= choice['epochs']: if step * batch_size > X_train.shape[0] or step == 0: s = np.arange(X_train.shape[0]) np.random.shuffle(s) X_train = X_train[s] y_train = y_train[s] step = 0 print "Epoch:%d, loss: %f, accuracy: %f" % (epoch, l, a) save_results(choice, teacher_model, student_model, loss, accuracy) epoch += 1 X_batch = X_train[step * batch_size:min(X_train.shape[0], (step + 1) * batch_size), :] y_batch = y_train[step * batch_size:min(y_train.shape[0], (step + 1) * batch_size), :] if choice['student'] != 'none': s = np.random.choice(X_batch.shape[0], X_batch.shape[0]) X_batch2 = X_batch[s] eps = np.random.rand(X_batch.shape[0])[:, None, None, None] eps = np.tile(eps, [1, 32, 32, 3]) X_batch3 = eps * X_batch + (1 - eps) * X_batch2 X_batch = np.concatenate( (X_batch[:X_batch.shape[0]], X_batch3[X_batch3.shape[0]:])) if choice['student'] != 'none': if choice['teacher'] == 'lnin_ssae': #X_batch, y_batch = teacher_model.correct(X_batch, y_batch) hidden = teacher_model.get_hidden(X_batch) hidden = ae.get_embedding(hidden) labels = ssae.get_embedding(hidden) elif choice['teacher'] == 'dnin': labels = teacher_model.predict(X_batch) #labels = y_batch elif choice['teacher'] == 'tnin': labels = teacher_model.get_embedding(X_batch) labels[labels >= 0.5] = 1 labels[labels < 0.5] = 0 elif choice['teacher'] == 'mnin': labels = teacher_model.get_sub_prediction(X_batch) elif choice['teacher'] == 'lnin': labels = teacher_model.get_logits(X_batch) l, a = student_model.train(X_batch, y_batch, labels) else: if choice['teacher'] == 'lnin_ae': #X_batch, y_batch = teacher_model.correct(X_batch, y_batch) hidden = teacher_model.get_hidden(X_batch) l, a = ae.train(hidden) elif choice['teacher'] == 'lnin_ssae': #X_batch, y_batch = teacher_model.correct(X_batch, y_batch) hidden = teacher_model.get_hidden(X_batch) hidden = ae.get_embedding(hidden) l, a = ssae.train(hidden, y_batch) else: l, a = teacher_model.train(X_batch, y_batch) loss.append(l) accuracy.append(a) if epoch == choice['epochs']: a = 0 for i in range(100): if choice['student'] != 'none': a += student_model.test(X_test[i * 100:(i + 1) * 100], y_test[i * 100:(i + 1) * 100]) else: a += teacher_model.test(X_test[i * 100:(i + 1) * 100], y_test[i * 100:(i + 1) * 100]) a = 1. * a / 100 print "Test Accuracy ", a test_accuracy.append(a) if step == 100: np.save("test_accuracy_"+choice['teacher']+'_'+choice['student']\ +"_run"+str(choice['run'])+".npy",np.asarray(test_accuracy)) print "Student Done" break step += 1
def data_aug(data, lr=0.001, epoch=800, batch_size=128): folder = 'data_aug' save_path = f'{folder}/data_augment.csv' clean_file(save_path) store_e = [700, 750, 800] if not os.path.exists(folder): os.makedirs(folder) else: for i in store_e: result = test(data, folder, i) return result train_loss_curve = [] valid_loss_curve = [] # load model device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = AE() model = model.to(device) model.train() dataset = AEDataset(data) train_size = int(0.85 * len(dataset)) valid_size = len(dataset) - train_size train_dataset, valid_dataset = random_split(dataset, [train_size, valid_size]) train_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) valid_dataloader = DataLoader(dataset=valid_dataset, batch_size=batch_size, shuffle=True) # loss function and optimizer # can change loss function and optimizer you want criterion = nn.MSELoss() optimizer = torch.optim.Adam(model.parameters(), lr=lr) best = 100 # start training for e in range(epoch): train_loss = 0.0 valid_loss = 0.0 print(f'\nEpoch: {e+1}/{epoch}') print('-' * len(f'Epoch: {e+1}/{epoch}')) # tqdm to disply progress bar for inputs in tqdm(train_dataloader): # data from data_loader inputs = inputs.float().to(device) outputs = model(inputs, device) loss = criterion(outputs, inputs) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() for inputs in tqdm(valid_dataloader): # data from data_loader inputs = inputs.float().to(device) outputs = model(inputs, device) # MSE loss loss = criterion(outputs, inputs) # loss calculate valid_loss += loss.item() # save the best model weights as .pth file loss_epoch = train_loss / len(train_dataset) valid_loss_epoch = valid_loss / len(valid_dataset) # if valid_loss_epoch < best : # best = valid_loss_epoch # torch.save(model.state_dict(), 'data_aug.pth') if e in store_e: torch.save(model.state_dict(), f'{folder}/ep{e}data_aug.pth') print(f"training in epoch {e},start augment data!!") result = test(data, folder, e) print(f'Training loss: {loss_epoch:.4f}') print(f'Valid loss: {valid_loss_epoch:.4f}') # save loss every epoch train_loss_curve.append(loss_epoch) valid_loss_curve.append(valid_loss_epoch) # generate training curve # visualize(train_loss_curve,valid_loss_curve, 'Data Augmentation') return result
feat_file = '/Volumes/behavgenom_archive$/Avelino/screening/CeNDR/Results/CeNDR_Set1_160517/BRC20067_worms10_food1-10_Set10_Pos5_Ch6_16052017_165021_featuresN.hdf5' if __name__ == '__main__': ae = AE(64) criterion = nn.MSELoss() optimizer = torch.optim.Adam(ae.parameters(), lr=1e-2) #%% #optimizer = torch.optim.SGD(vae.parameters(), lr=1e-1, momentum=0.9) n_epochs = 100 #%% gen = ROIFlowBatch(mask_file, feat_file, batch_size=32, roi_size=128) ae.train() for epoch in range(n_epochs): pbar = tqdm.tqdm(gen) for X in pbar: decoded = ae.forward(X) loss = criterion(decoded, X) # mean square error optimizer.zero_grad() # clear gradients for this training step loss.backward() # backpropagation, compute gradients optimizer.step() dd = 'Epoch {} , loss={}'.format(epoch, loss.data[0]) pbar.set_description(desc=dd, refresh=False) #%% cx = X.data.numpy() decoded = ae.forward(X) imgs = decoded.data.numpy()