def pre_train(model, optimizer, dl_train, dl_test, x_train, y_train, run, maxEpoch=200): losses = [] accuracies = [] losses_validate = [] accuracies_validate = [] bestValidationAcc = 0. alpha = 0. r2 = 0. for epoch in range(maxEpoch): for i, batch in enumerate(dl_train): loss, x, y, y_pred, z = step(model, optimizer, batch, alpha, r2) accuracy, loss = eval_step(model, x_train, y_train, alpha, r2) losses.append(loss) accuracies.append(accuracy) if (epoch+1) % 100 == 0: print('train: epoch {}, accuracy {:.4f}, loss {:.4f}'.format(epoch+1, accuracy, loss)) accuracy, loss = eval_step(model, x_validate, y_validate, alpha, r2) if accuracy > bestValidationAcc: # stateDict = model.state_dict() # include both learnable param and registered buffer PATH = '/home/hh/data/csnn_2_layers_run{}_epoch{}.pth'.format(run, 0) torch.save({'net': model, 'alpha': alpha, 'r2': r2}, PATH) bestValidationAcc = accuracy if (epoch+1) % 100 == 0: print('validation: epoch {}, accuracy {:.4f}, loss {:.4f}'.format(epoch+1, accuracy, loss)) losses_validate.append(loss) accuracies_validate.append(accuracy) plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_pretrain_2_layers_run{}.png'.format(run)) plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_pretrain_2_layers_run{}.png'.format(run)) print('Done pre-train, best validation acc {:.4f}'.format(bestValidationAcc))
step(model, optimizer, batch) accuracy, loss = eval_step(model, x_train, y_train) testacc, testloss = eval_step(model, x_validate, y_validate) #if epoch % 10 == 0: # print('train: epoch {}, accuracy {:.3f}, loss {:.3f}'.format(epoch, accuracy, loss)) if testacc > bestValidationAcc: # include both learnable param and registered buffer # PATH = outputDir+'/csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format(run, r2, maxAlpha) # torch.save({'net':model, 'alpha':alpha, 'r2':r2}, PATH) bestValidationAcc = testacc if epoch % 5 == 0: #print('validation: epoch {}, fc1 shape {}, loss {:.3f}'.format(epoch, model.fc1.weight.data.shape[0], loss)) losses.append(loss) losses_validate.append(testloss) accuracies.append(accuracy) accuracies_validate.append(testacc) print('epoch {}, train {:.3f}, test {:.3f}'.format(epoch,accuracy,testacc)) plot_save_loss(losses, losses_validate, outputDir+'/loss_run{}.png'.format(run)) plot_save_acc(accuracies, accuracies_validate, outputDir+'/acc_run{}.png'.format(run)) bestValidationAccs.append(max(accuracies_validate)) ACCs.append(accuracies_validate) ACCs = np.array(ACCs) print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}' .format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs)))) dir = outputDir + '/mean_std_accs_aucs_net4.npz' np.savez(dir, a=np.mean(ACCs, axis=0), b=np.std(ACCs, axis=0)) # plt.show()
e=np.array(nzs), f=np.array(mmcs)) # plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}.png'.format(run)) # plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}.png'.format(run)) # plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, # '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}.png'.format(run)) # plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_trim.png'.format(run)) # plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_trim.png'.format(run)) # plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, # '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_trim.png'.format(run)) # plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2)) # plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2)) # plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, # '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2)) plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha)) plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha)) plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha)) bestValidationAccs.append(max(accuracies_validate)) AUCs.append(aucs) ACCs.append(accs) print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}' .format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs)))) # plt.show() AUCs = np.array(AUCs) ACCs = np.array(ACCs) print('average and std of AUC in {} runs'.format(runs)) # print('mean ', np.mean(AUCs, axis=0)) # print('std ', np.std(AUCs, axis=0))
_, dmu = active(torch.tensor(x_train).float(), model, alpha, r2) print('fc3 non zero neurons {}'.format(np.sum(dmu>0))) # model.keepNodes(dmu > 0) # print('epoch ', epoch) dir0 = '/home/hh/data/' dir = '/home/hh/data/loss_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}'.format(maxAlpha, run) np.savez(dir + ".npz", a=np.array(losses), b=np.array(losses_validate), c=np.array(mmcs)) dir = '/home/hh/data/acc_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}'.format(maxAlpha, run) np.savez(dir + ".npz", a=np.array(accuracies), b=np.array(accuracies_validate)) dir = '/home/hh/data/csnn_nz_mmcs_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.npz'.format(maxAlpha, run) np.savez(dir, a=np.array(nzs), b=np.array(mmcs)) plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run)) plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run)) plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, '/home/hh/data/acc_nzs_mmcs_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run)) bestValidationAccs.append(max(accuracies_validate)) print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}' .format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs)))) # plt.show() else: AUCs = [] ACCs = [] epochs = [] # for i in range(10, 191, 10): for i in range(10, 101, 10): epochs.append(i)
# eliminate dead nodes if ((epoch < 200 and (epoch + 1) % (epochs / 100) == 0) or (epoch >= 200 and (epoch + 1) % (epochs / 10) == 0)): _, dmu = active( torch.tensor(x_train).float(), model, alpha, r2) PATH = outputDir + '/csnn_2_csnn_layers_run{}_epoch{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format( run, epoch + 1, r2, maxAlpha) torch.save({'net': model, 'alpha': alpha, 'r2': r2}, PATH) # model.keepNodes(dmu > 0) plot_save_loss( losses, losses_validate, outputDir + '/loss_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png' .format(run, r2, maxAlpha)) plot_save_acc( accuracies, accuracies_validate, outputDir + '/acc_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png' .format(run, r2, maxAlpha)) plot_save_acc_nzs_mmcs( alphas, accuracies_validate, nzs, mmcs, outputDir + '/acc_nzs_mmcs_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png' .format(run, r2, maxAlpha)) bestValidationAccs.append(max(accuracies_validate)) print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'. format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs)))) # plt.show() else: AUCs = [] ACCs = []
# eliminate dead nodes # if ( (epoch<200 and (epoch + 1) % (epochs / 100) == 0) # or (epoch>=200 and (epoch + 1) % (epochs/10) == 0)): # #_, dmu = active(torch.tensor(x_train).float(), model, alpha, r2) # PATH = outputDir+'/csnn_2_csnn_layers_run{}_epoch{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format(run, epoch+1, r2, maxAlpha) # torch.save({'net':model, 'alpha':alpha, 'r2':r2}, PATH) # # model.keepNodes(dmu > 0) # plot_save_loss(losses, losses_validate, outputDir+'/loss_run{}.png'.format(run)) plot_save_loss( losses, losses_validate, outputDir + '/loss_lambda{:.2f}_run{}.png'.format(LAMBDA, run)) # plot_save_acc(accuracies, accuracies_validate, outputDir+'/acc_run{}.png'.format(run)) plot_save_acc( accuracies, accuracies_validate, outputDir + '/acc_lambda{:.2f}_run{}.png'.format(LAMBDA, run)) plot_save_acc_nzs_mmcs( alphas, accuracies_validate, nzs, aucs, # outputDir+'/acc_nzs_mmcs_run{}.png'.format(run)) outputDir + '/acc_nzs_mmcs_lambda{:.2f}_run{}.png'.format(LAMBDA, run)) bestValidationAccs.append(max(accuracies_validate)) AUCs.append(aucs) ACCs.append(accuracies_validate) if ALPHAs is None: ALPHAs = alphas AUCs = np.array(AUCs)
if epoch % (epochs/10) == 0: _, dmu = active(torch.tensor(x_train[:, mask]).float(), model, alpha, r2) print('fc3 non zero neurons {}'.format(np.sum(dmu>0))) # model.keepNodes(dmu > 0) # print('epoch ', epoch) dir = '/home/hh/data/loss_csnn' np.savez(dir + ".npz", a=np.array(losses), b=np.array(losses_validate), c=np.array(mmcs)) dir = '/home/hh/data/acc_csnn' np.savez(dir + ".npz", a=np.array(accuracies), b=np.array(accuracies_validate)) dir = '/home/hh/data/csnn_nz_mmcs.npz' np.savez(dir, a=np.array(nzs), b=np.array(mmcs)) plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn.png') plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn.png') plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, '/home/hh/data/acc_nzs_mmcs_csnn.png') bestValidationAccs.append(max(accuracies_validate)) print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}' .format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs)))) # plt.show() else: AUCs = [] for run in range(runs): PATH = '/home/hh/data/csnn_run{}.pth'.format(run) l = torch.load(PATH) model = l['net'] alpha = l['alpha'] r2 = l['r2']