def pre_train(model, optimizer, dl_train, dl_test, x_train, y_train, run, maxEpoch=200):
losses = []
accuracies = []
losses_validate = []
accuracies_validate = []
bestValidationAcc = 0.
alpha = 0.
r2 = 0.
for epoch in range(maxEpoch):
for i, batch in enumerate(dl_train):
loss, x, y, y_pred, z = step(model, optimizer, batch, alpha, r2)
accuracy, loss = eval_step(model, x_train, y_train, alpha, r2)
losses.append(loss)
accuracies.append(accuracy)
if (epoch+1) % 100 == 0:
print('train: epoch {}, accuracy {:.4f}, loss {:.4f}'.format(epoch+1, accuracy, loss))
accuracy, loss = eval_step(model, x_validate, y_validate, alpha, r2)
if accuracy > bestValidationAcc:
# stateDict = model.state_dict()
# include both learnable param and registered buffer
PATH = '/home/hh/data/csnn_2_layers_run{}_epoch{}.pth'.format(run, 0)
torch.save({'net': model, 'alpha': alpha, 'r2': r2}, PATH)
bestValidationAcc = accuracy
if (epoch+1) % 100 == 0:
print('validation: epoch {}, accuracy {:.4f}, loss {:.4f}'.format(epoch+1, accuracy, loss))
losses_validate.append(loss)
accuracies_validate.append(accuracy)
plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_pretrain_2_layers_run{}.png'.format(run))
plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_pretrain_2_layers_run{}.png'.format(run))
print('Done pre-train, best validation acc {:.4f}'.format(bestValidationAcc))
step(model, optimizer, batch)
accuracy, loss = eval_step(model, x_train, y_train)
testacc, testloss = eval_step(model, x_validate, y_validate)
#if epoch % 10 == 0:
# print('train: epoch {}, accuracy {:.3f}, loss {:.3f}'.format(epoch, accuracy, loss))
if testacc > bestValidationAcc:
# include both learnable param and registered buffer
# PATH = outputDir+'/csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format(run, r2, maxAlpha)
# torch.save({'net':model, 'alpha':alpha, 'r2':r2}, PATH)
bestValidationAcc = testacc
if epoch % 5 == 0:
#print('validation: epoch {}, fc1 shape {}, loss {:.3f}'.format(epoch, model.fc1.weight.data.shape[0], loss))
losses.append(loss)
losses_validate.append(testloss)
accuracies.append(accuracy)
accuracies_validate.append(testacc)
print('epoch {}, train {:.3f}, test {:.3f}'.format(epoch,accuracy,testacc))
plot_save_loss(losses, losses_validate, outputDir+'/loss_run{}.png'.format(run))
plot_save_acc(accuracies, accuracies_validate, outputDir+'/acc_run{}.png'.format(run))
bestValidationAccs.append(max(accuracies_validate))
ACCs.append(accuracies_validate)
ACCs = np.array(ACCs)
print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'
.format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs))))
dir = outputDir + '/mean_std_accs_aucs_net4.npz'
np.savez(dir, a=np.mean(ACCs, axis=0), b=np.std(ACCs, axis=0))
# plt.show()
e=np.array(nzs), f=np.array(mmcs))
# plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}.png'.format(run))
# plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}.png'.format(run))
# plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs,
# '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}.png'.format(run))
# plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_trim.png'.format(run))
# plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_trim.png'.format(run))
# plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs,
# '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_trim.png'.format(run))
# plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2))
# plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2))
# plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs,
# '/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_r2{:.1f}.png'.format(run, r2))
plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha))
plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha))
plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs,
'/home/hh/data/acc_nzs_mmcs_csnn_2_layers_run{}_r2{:.1f}_maxAlpha{:.1f}.png'.format(run, r2, maxAlpha))
bestValidationAccs.append(max(accuracies_validate))
AUCs.append(aucs)
ACCs.append(accs)
print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'
.format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs))))
# plt.show()
AUCs = np.array(AUCs)
ACCs = np.array(ACCs)
print('average and std of AUC in {} runs'.format(runs))
# print('mean ', np.mean(AUCs, axis=0))
# print('std ', np.std(AUCs, axis=0))
_, dmu = active(torch.tensor(x_train).float(), model, alpha, r2)
print('fc3 non zero neurons {}'.format(np.sum(dmu>0)))
# model.keepNodes(dmu > 0)
# print('epoch ', epoch)
dir0 = '/home/hh/data/'
dir = '/home/hh/data/loss_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}'.format(maxAlpha, run)
np.savez(dir + ".npz", a=np.array(losses), b=np.array(losses_validate), c=np.array(mmcs))
dir = '/home/hh/data/acc_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}'.format(maxAlpha, run)
np.savez(dir + ".npz", a=np.array(accuracies), b=np.array(accuracies_validate))
dir = '/home/hh/data/csnn_nz_mmcs_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.npz'.format(maxAlpha, run)
np.savez(dir, a=np.array(nzs), b=np.array(mmcs))
plot_save_loss(losses, losses_validate,
'/home/hh/data/loss_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run))
plot_save_acc(accuracies, accuracies_validate,
'/home/hh/data/acc_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run))
plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs,
'/home/hh/data/acc_nzs_mmcs_csnn_4_layers_batchNorm_noPretrain_maxAlpha{:.2f}_run{}.png'.format(maxAlpha, run))
bestValidationAccs.append(max(accuracies_validate))
print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'
.format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs))))
# plt.show()
else:
AUCs = []
ACCs = []
epochs = []
# for i in range(10, 191, 10):
for i in range(10, 101, 10):
epochs.append(i)
# eliminate dead nodes
if ((epoch < 200 and (epoch + 1) % (epochs / 100) == 0) or
(epoch >= 200 and (epoch + 1) % (epochs / 10) == 0)):
_, dmu = active(
torch.tensor(x_train).float(), model, alpha, r2)
PATH = outputDir + '/csnn_2_csnn_layers_run{}_epoch{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format(
run, epoch + 1, r2, maxAlpha)
torch.save({'net': model, 'alpha': alpha, 'r2': r2}, PATH)
# model.keepNodes(dmu > 0)
plot_save_loss(
losses, losses_validate, outputDir +
'/loss_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png'
.format(run, r2, maxAlpha))
plot_save_acc(
accuracies, accuracies_validate, outputDir +
'/acc_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png'
.format(run, r2, maxAlpha))
plot_save_acc_nzs_mmcs(
alphas, accuracies_validate, nzs, mmcs, outputDir +
'/acc_nzs_mmcs_csnn_2_csnn_layers_run{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.png'
.format(run, r2, maxAlpha))
bestValidationAccs.append(max(accuracies_validate))
print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'.
format(runs, np.mean(np.array(bestValidationAccs)),
np.std(np.array(bestValidationAccs))))
# plt.show()
else:
AUCs = []
ACCs = []
# eliminate dead nodes
# if ( (epoch<200 and (epoch + 1) % (epochs / 100) == 0)
# or (epoch>=200 and (epoch + 1) % (epochs/10) == 0)):
# #_, dmu = active(torch.tensor(x_train).float(), model, alpha, r2)
# PATH = outputDir+'/csnn_2_csnn_layers_run{}_epoch{}_r2{:.1f}_maxAlpha{:.1f}_affine_false.pth'.format(run, epoch+1, r2, maxAlpha)
# torch.save({'net':model, 'alpha':alpha, 'r2':r2}, PATH)
# # model.keepNodes(dmu > 0)
# plot_save_loss(losses, losses_validate, outputDir+'/loss_run{}.png'.format(run))
plot_save_loss(
losses, losses_validate,
outputDir + '/loss_lambda{:.2f}_run{}.png'.format(LAMBDA, run))
# plot_save_acc(accuracies, accuracies_validate, outputDir+'/acc_run{}.png'.format(run))
plot_save_acc(
accuracies, accuracies_validate,
outputDir + '/acc_lambda{:.2f}_run{}.png'.format(LAMBDA, run))
plot_save_acc_nzs_mmcs(
alphas,
accuracies_validate,
nzs,
aucs,
# outputDir+'/acc_nzs_mmcs_run{}.png'.format(run))
outputDir +
'/acc_nzs_mmcs_lambda{:.2f}_run{}.png'.format(LAMBDA, run))
bestValidationAccs.append(max(accuracies_validate))
AUCs.append(aucs)
ACCs.append(accuracies_validate)
if ALPHAs is None: ALPHAs = alphas
AUCs = np.array(AUCs)
if epoch % (epochs/10) == 0:
_, dmu = active(torch.tensor(x_train[:, mask]).float(), model, alpha, r2)
print('fc3 non zero neurons {}'.format(np.sum(dmu>0)))
# model.keepNodes(dmu > 0)
# print('epoch ', epoch)
dir = '/home/hh/data/loss_csnn'
np.savez(dir + ".npz", a=np.array(losses), b=np.array(losses_validate), c=np.array(mmcs))
dir = '/home/hh/data/acc_csnn'
np.savez(dir + ".npz", a=np.array(accuracies), b=np.array(accuracies_validate))
dir = '/home/hh/data/csnn_nz_mmcs.npz'
np.savez(dir, a=np.array(nzs), b=np.array(mmcs))
plot_save_loss(losses, losses_validate, '/home/hh/data/loss_csnn.png')
plot_save_acc(accuracies, accuracies_validate, '/home/hh/data/acc_csnn.png')
plot_save_acc_nzs_mmcs(alphas, accuracies_validate, nzs, mmcs, '/home/hh/data/acc_nzs_mmcs_csnn.png')
bestValidationAccs.append(max(accuracies_validate))
print('mean and std of best validation acc in {} runs: {:.4f}, {:.4f}'
.format(runs, np.mean(np.array(bestValidationAccs)), np.std(np.array(bestValidationAccs))))
# plt.show()
else:
AUCs = []
for run in range(runs):
PATH = '/home/hh/data/csnn_run{}.pth'.format(run)
l = torch.load(PATH)
model = l['net']
alpha = l['alpha']
r2 = l['r2']
