def run(model) -> typing.Tuple: """Method train and validate model""" model.train() optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) train_lost = [] val_lost = [] train_acc = [] val_acc = [] for epoch in range(200): optimizer.zero_grad() pred = model(adj_matrix, nodes_features, deg) loss_train = F.nll_loss(pred[train_index], labels[train_index]) acc_train = accuracy(pred[train_index], labels[train_index]) loss_train.backward() optimizer.step() model.eval() loss_val = F.nll_loss(pred[val_index], labels[val_index]) acc_val = accuracy(pred[val_index], labels[val_index]) if epoch % 10 == 0: print("epoch", epoch, end=" ") print("train loss", loss_train.item(), end=" ") print("val loss", loss_val.item(), end=" ") print("train accu", acc_train.item(), end=" ") print("val accu", acc_val.item(), end=" ") print() train_acc.append(acc_train.item()) val_acc.append(acc_val.item()) train_lost.append(loss_train.item()) val_lost.append(loss_val.item()) return val_acc, val_lost, train_acc, train_lost
def test(configer): ## datasets testset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'test', configer.usedChannels) testloader = DataLoader(testset, configer.batchsize_test, shuffle=False) ## model modelpath = os.path.join(configer.mdlspath, configer.modelname) + '.pkl' assert os.path.exists(modelpath), 'please train first! ' model = torch.load(modelpath) if configer.cuda and is_available(): model.cuda() ## loss loss = nn.CrossEntropyLoss() ## log logpath = os.path.join(configer.logspath, configer.modelname) ftest = open(os.path.join(logpath, 'test_log.txt'), 'w') ## initialize acc_test = [] loss_test = [] output = None ArcMargin = ArcMarginProduct(128, configer.n_class) ## start testing model.eval() for i_batch, (X, y) in enumerate(testloader): # get batch X = Variable(X.float()) y = Variable(y) if configer.cuda and is_available(): X = X.cuda() y = y.cuda() # forward if configer.modelbase == 'recognize_mobilefacenet': raw_logits = model(X) y_pred_prob = ArcMargin(raw_logits, y) else: y_pred_prob = model(X) #y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # log print_log = "{} || Batch: [{:3d}]/[{:3d}] || accuracy: {:2.2%}, loss: {:4.4f}".\ format(getTime(), i_batch, len(testset) // configer.batchsize, acc_i, loss_i) print(print_log) ftest.write(print_log + '\n') loss_test += [loss_i.detach().cpu().numpy()] acc_test += [acc_i.cpu().numpy()] # save output if output is None: output = y_pred_prob.detach().cpu().numpy() else: output = np.concatenate( [output, y_pred_prob.detach().cpu().numpy()], axis=0) print( '------------------------------------------------------------------------------------------------------------------' ) loss_test = np.mean(np.array(loss_test)) acc_test = np.mean(np.array(acc_test)) print_log = "{} || test | acc: {:2.2%}, loss: {:4.4f}".\ format(getTime(), acc_test, loss_test) print(print_log) ftest.write(print_log + '\n') np.save(os.path.join(logpath, 'test_out.npy'), output) print( '==================================================================================================================' ) ftest.close()
def test(): batch_size = configer.batchsize modelname = configer.modelname splitmode = configer.splitmode testsets = HyperECUST(splitmode, configer.facesize, 'test') testloader = DataLoader(testsets, batch_size) model, modelpath = init_model() print_log = 'load model: {}'.format(modelpath) print(print_log) loss = init_loss() acc_test = [] loss_test = [] model.eval() output_tosave = None txtfile = os.path.join(configer.logspath, modelname, 'test_result_{}.txt'.format(splitmode)) f = open(txtfile, 'w') for i_batch, (X, y) in enumerate(testloader): X = Variable(X.float()) if torch.cuda.is_available(): X = X.cuda() y = y.cuda() y_pred_prob = model(X) loss_test_batch = loss(y_pred_prob, y) acc_test_batch = accuracy(y_pred_prob, y, multi=False) print_log = 'testing... batch [{:2d}]/[{:2d}] || accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_batch+1, len(testsets)//batch_size, acc_test_batch, loss_test_batch) print(print_log) f.write(print_log + '\n') loss_test.append(loss_test_batch.detach().cpu().numpy()) acc_test.append(acc_test_batch.cpu().numpy()) # save outputs output = y_pred_prob.detach().cpu().numpy() if i_batch == 0: output_tosave = output else: output_tosave = np.concatenate([output_tosave, output], axis=0) acc_test = np.mean(np.array(acc_test)) loss_test = np.mean(np.array(loss_test)) print( '--------------------------------------------------------------------') print_log = "{} || accuracy: {:2.2%}, loss: {:4.4f}".\ format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), acc_test, loss_test) print(print_log) f.write(print_log + '\n') npyfile = os.path.join(configer.logspath, modelname, 'test_output_{}.npy'.format(splitmode)) np.save(npyfile, output_tosave) f.close() print( '====================================================================')
def train(configer): ## datasets trainset = AnalysisDataset(configer.datapath, configer.splitmode, 'train') validset = AnalysisDataset(configer.datapath, configer.splitmode, 'valid') trainloader = DataLoader(trainset, configer.batchsize, shuffle=True) validloader = DataLoader(validset, configer.batchsize, shuffle=True) ## model modelpath = os.path.join(configer.mdlspath, configer.modelname) + '.pkl' if not os.path.exists(configer.mdlspath): os.makedirs(configer.mdlspath) model = modeldict[configer.modelbase](1, configer.n_class, configer.dsize[0]) if configer.cuda and is_available(): model.cuda() ## loss loss = nn.CrossEntropyLoss() ## optimizer params = model.parameters() optimizer = optim.Adam(params, configer.lrbase, weight_decay=5e-4) ## learning rate scheduler scheduler = lr_scheduler.StepLR(optimizer, configer.stepsize, configer.gamma) ## log logpath = os.path.join(configer.logspath, configer.modelname) if not os.path.exists(logpath): os.makedirs(logpath) logger = SummaryWriter(logpath) ## initialize elapsed_time = 0 total_time = 0 start_time = 0 acc_train = 0. acc_valid = 0. loss_train = float('inf') loss_valid = float('inf') loss_valid_last = float('inf') ## start training for i_epoch in range(configer.n_epoch): if configer.cuda and is_available(): empty_cache() scheduler.step(i_epoch) acc_train = [] acc_valid = [] loss_train = [] loss_valid = [] model.train() start_time = time.time() for i_batch, (X, y) in enumerate(trainloader): # get batch X = Variable(X.float()) y = Variable(y) if configer.cuda and is_available(): X = X.cuda() y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # backward optimizer.zero_grad() loss_i.backward() optimizer.step() # time duration_time = time.time() - start_time start_time = time.time() elapsed_time += duration_time total_time = duration_time * configer.n_epoch * len( trainset) // configer.batchsize # log print_log = "{} || Elapsed: {:.4f}h | Left: {:.4f}h | FPS: {:4.2f} || Epoch: [{:3d}]/[{:3d}] | Batch: [{:3d}]/[{:3d}] || lr: {:.6f} | accuracy: {:2.2%}, loss: {:4.4f}".\ format(getTime(), elapsed_time/3600, (total_time - elapsed_time)/3600, configer.batchsize / duration_time, i_epoch, configer.n_epoch, i_batch, len(trainset) // configer.batchsize, scheduler.get_lr()[-1], acc_i, loss_i) print(print_log) loss_train += [loss_i.detach().cpu().numpy()] acc_train += [acc_i.cpu().numpy()] print( '------------------------------------------------------------------------------------------------------------------' ) model.eval() for i_batch, (X, y) in enumerate(validloader): # get batch X = Variable(X.float()) y = Variable(y) if configer.cuda and is_available(): X = X.cuda() y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # log print_log = "{} || Epoch: [{:3d}]/[{:3d}] | Batch: [{:3d}]/[{:3d}] || accuracy: {:2.2%}, loss: {:4.4f}".\ format(getTime(), i_epoch, configer.n_epoch, i_batch, len(validset) // configer.batchsize, acc_i, loss_i) print(print_log) loss_valid += [loss_i.detach().cpu().numpy()] acc_valid += [acc_i.cpu().numpy()] print( '------------------------------------------------------------------------------------------------------------------' ) loss_train = np.mean(np.array(loss_train)) acc_train = np.mean(np.array(acc_train)) loss_valid = np.mean(np.array(loss_valid)) acc_valid = np.mean(np.array(acc_valid)) print_log = "{} || Epoch: [{:3d}]/[{:3d}] || lr: {:.6f} || train | acc: {:2.2%}, loss: {:4.4f} || valid | acc: {:2.2%}, loss: {:4.4f}".\ format(getTime(), i_epoch, configer.n_epoch, scheduler.get_lr()[-1], acc_train, loss_train, acc_valid, loss_valid) print(print_log) logger.add_scalars('accuracy', { 'train': acc_train, 'valid': acc_valid }, i_epoch) logger.add_scalars('logloss', { 'train': loss_train, 'valid': loss_valid }, i_epoch) logger.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) print( '------------------------------------------------------------------------------------------------------------------' ) if loss_valid_last > loss_valid: loss_valid_last = loss_valid torch.save(model, modelpath) print_log = "{} || Epoch: [{:3d}]/[{:3d}] || Saved as {}".\ format(getTime(), i_epoch, configer.n_epoch, modelpath) print(print_log) print( '==================================================================================================================' )
def main_pca(): from tensorPCA import NDarrayPCA for splitidx in range(1, 6): configer = EasyDict() configer.dsize = (64, 64) configer.datatype = 'Multi' configer.n_epoch = 500 configer.lrbase = 0.0001 configer.n_channel = 23 configer.n_class = 63 configer.batchsize = 32 configer.stepsize = 250 configer.gamma = 0.2 configer.cuda = True configer.splitmode = 'split_{}x{}_{}'.format(configer.dsize[0], configer.dsize[1], splitidx) configer.modelbase = 'recognize_vgg11_bn' configer.usedChannels = [550+i*20 for i in range(23)] configer.n_usedChannels = len(configer.usedChannels) configer.modelname = '{}_{}_{}_PCA'.\ format(configer.modelbase, configer.splitmode, '_'.join(list(map(str, configer.usedChannels)))) configer.datapath = '/home/louishsu/Work/Workspace/ECUST2019_{}x{}'.\ format(configer.dsize[0], configer.dsize[1]) configer.logspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/logs/{}_{}_{}subjects_logs'.\ format(configer.modelbase, configer.splitmode, configer.n_class) configer.mdlspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/modelfiles/{}_{}_{}subjects_models'.\ format(configer.modelbase, configer.splitmode, configer.n_class) trainset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'train', configer.usedChannels) validset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'valid', configer.usedChannels) trainloader = DataLoader(trainset, configer.batchsize, shuffle=True) validloader = DataLoader(validset, configer.batchsize, shuffle=False) for chs in range(10, 24): print(getTime(), splitidx, 'reduce to ', chs, '...') ## fit pca decomposer = NDarrayPCA(n_components=[chs, 64, 64]) traindata = np.concatenate([trainset.samplelist[i][0].numpy()[np.newaxis] for i in range(len(trainset.samplelist))], axis=0) decomposer.fit(traindata) del traindata ## model modelpath = os.path.join(configer.mdlspath, configer.modelname) + '{}chs.pkl'.format(chs) modeldir = '/'.join(modelpath.split('/')[:-1]) if not os.path.exists(modeldir): os.makedirs(modeldir) model = modeldict[configer.modelbase](chs, configer.n_class, configer.dsize[0]) if configer.cuda and is_available(): model.cuda() ## loss loss = nn.CrossEntropyLoss() params = model.parameters() optimizer = optim.Adam(params, configer.lrbase, weight_decay=1e-3) scheduler = lr_scheduler.StepLR(optimizer, configer.stepsize, configer.gamma) logpath = os.path.join(configer.logspath, configer.modelname) + '{}chs'.format(chs) if not os.path.exists(logpath): os.makedirs(logpath) logger = SummaryWriter(logpath) ## initialize acc_train = 0. acc_valid = 0. loss_train = float('inf') loss_valid = float('inf') loss_valid_last = float('inf') ## start training for i_epoch in range(configer.n_epoch): if configer.cuda and is_available(): empty_cache() scheduler.step(i_epoch) acc_train = []; acc_valid = [] loss_train = []; loss_valid = [] model.train() for i_batch, (X, y) in enumerate(trainloader): X = torch.from_numpy(decomposer.transform(X.numpy())) # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # backward optimizer.zero_grad() loss_i.backward() optimizer.step() loss_train += [loss_i.detach().cpu().numpy()] acc_train += [acc_i.cpu().numpy()] model.eval() for i_batch, (X, y) in enumerate(validloader): X = torch.from_numpy(decomposer.transform(X.numpy())) # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) loss_valid += [loss_i.detach().cpu().numpy()] acc_valid += [acc_i.cpu().numpy()] loss_train = np.mean(np.array(loss_train)) acc_train = np.mean(np.array(acc_train)) loss_valid = np.mean(np.array(loss_valid)) acc_valid = np.mean(np.array(acc_valid)) logger.add_scalars('accuracy', {'train': acc_train, 'valid': acc_valid}, i_epoch) logger.add_scalars('logloss', {'train': loss_train, 'valid': loss_valid}, i_epoch) logger.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) if loss_valid_last > loss_valid: loss_valid_last = loss_valid torch.save(model, modelpath) ## start testing model.eval() testset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'test', configer.usedChannels) testloader = DataLoader(testset, configer.batchsize, shuffle=False) loss_test = [] acc_test = [] output = None for i_batch, (X, y) in enumerate(testloader): X = torch.from_numpy(decomposer.transform(X.numpy())) # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # log loss_test += [loss_i.detach().cpu().numpy()] acc_test += [acc_i.cpu().numpy()] # save output if output is None: output = y_pred_prob.detach().cpu().numpy() else: output = np.concatenate([output, y_pred_prob.detach().cpu().numpy()], axis=0) # print('------------------------------------------------------------------------------------------------------------------') loss_test = np.mean(np.array(loss_test)) acc_test = np.mean(np.array(acc_test)) print_log = "{} || test | acc: {:2.2%}, loss: {:4.4f}".\ format(getTime(), acc_test, loss_test) print(print_log) with open(os.path.join(logpath, 'test_log.txt'), 'w') as f: f.write(print_log + '\n') np.save(os.path.join(logpath, 'test_out.npy'), output)
def main_finetune_channels(): # 波段选择依据 # 最优的波段排序: # [850, 870, 930, 730, 790, 910, 770, 750, 670, 950, 990, 830, 890, 810, 970, 690, 710, 650, 590, 570, 630, 610, 550] # 依次增加一个波段, 前一个模型进行微调 CHANNEL_SORT = [850, 870, 930, 730, 790, 910, 770, 750, 670, 950, 990, 830, 890, 810, 970, 690, 710, 650, 590, 570, 630, 610, 550] for splitidx in range(4, 5): usedChannelsList = [CHANNEL_SORT[:i+1] for i in range(23)] # for i_usedChannels in range(len(usedChannelsList)): for i_usedChannels in [4, 6]: usedChannels = usedChannelsList[i_usedChannels] print(getTime(), splitidx, len(usedChannels), '...') configer = EasyDict() configer.dsize = (64, 64) configer.datatype = 'Multi' configer.n_epoch = 300 configer.lrbase = 0.001 configer.n_channel = 23 configer.n_class = 63 configer.batchsize = 32 configer.stepsize = 250 configer.gamma = 0.2 configer.cuda = True configer.splitmode = 'split_{}x{}_{}'.format(configer.dsize[0], configer.dsize[1], splitidx) configer.modelbase = 'recognize_mobilenet' configer.usedChannels = usedChannels configer.n_usedChannels = len(configer.usedChannels) configer.modelname = '{}_{}_{}_finetune'.\ format(configer.modelbase, configer.splitmode, '_'.join(list(map(str, configer.usedChannels)))) configer.datapath = '/home/louishsu/Work/Workspace/ECUST2019_{}x{}'.\ format(configer.dsize[0], configer.dsize[1]) configer.logspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/logs/{}_{}_{}subjects_logs'.\ format(configer.modelbase, configer.splitmode, configer.n_class) configer.mdlspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/modelfiles/{}_{}_{}subjects_models'.\ format(configer.modelbase, configer.splitmode, configer.n_class) ## datasets trainset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'train', configer.usedChannels) validset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'valid', configer.usedChannels) trainloader = DataLoader(trainset, configer.batchsize, shuffle=True) validloader = DataLoader(validset, configer.batchsize, shuffle=False) ## ============================================================================================ ## model modelpath = os.path.join(configer.mdlspath, configer.modelname) + '.pkl' modeldir = '/'.join(modelpath.split('/')[:-1]) if not os.path.exists(modeldir): os.makedirs(modeldir) if i_usedChannels == 0: model = modeldict[configer.modelbase](configer.n_usedChannels, configer.n_class, configer.dsize[0]) params = model.parameters() torch.save(model, modelpath) else: modelpath_pretrain = os.path.join( modeldir, '{}_{}_{}_finetune.pkl'.format(configer.modelbase, configer.splitmode, '_'.join(list(map(str, usedChannelsList[i_usedChannels-1]))))) model = torch.load(modelpath_pretrain) model.features[0] = nn.Conv2d(len(usedChannels), 64, 3, stride=1, padding=1) params = [ {'params': model.features[1:].parameters(), 'lr': configer.lrbase*0.01, }, {'params': model.features[0].parameters(),} ] torch.save(model, modelpath) if configer.cuda and is_available(): model.cuda() ## ============================================================================================ ## optimizer optimizer = optim.Adam(params, configer.lrbase, weight_decay=1e-3) ## loss loss = nn.CrossEntropyLoss() ## learning rate scheduler scheduler = lr_scheduler.StepLR(optimizer, configer.stepsize, configer.gamma) ## log logpath = os.path.join(configer.logspath, configer.modelname) if not os.path.exists(logpath): os.makedirs(logpath) logger = SummaryWriter(logpath) ## initialize acc_train = 0. acc_valid = 0. loss_train = float('inf') loss_valid = float('inf') loss_valid_last = float('inf') ## start training for i_epoch in range(configer.n_epoch): if configer.cuda and is_available(): empty_cache() scheduler.step(i_epoch) acc_train = []; acc_valid = [] loss_train = []; loss_valid = [] model.train() for i_batch, (X, y) in enumerate(trainloader): # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # backward optimizer.zero_grad() loss_i.backward() optimizer.step() loss_train += [loss_i.detach().cpu().numpy()] acc_train += [acc_i.cpu().numpy()] model.eval() for i_batch, (X, y) in enumerate(validloader): # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) loss_valid += [loss_i.detach().cpu().numpy()] acc_valid += [acc_i.cpu().numpy()] loss_train = np.mean(np.array(loss_train)) acc_train = np.mean(np.array(acc_train)) loss_valid = np.mean(np.array(loss_valid)) acc_valid = np.mean(np.array(acc_valid)) logger.add_scalars('accuracy', {'train': acc_train, 'valid': acc_valid}, i_epoch) logger.add_scalars('logloss', {'train': loss_train, 'valid': loss_valid}, i_epoch) logger.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) if loss_valid_last > loss_valid: loss_valid_last = loss_valid torch.save(model, modelpath) test(configer)
def main_several_channels_k_fold(k=5): # 波段选择依据 # 最优的波段排序: # [850, 870, 930, 730, 790, 910, 770, 750, 670, 950, 990, 830, 890, 810, 970, 690, 710, 650, 590, 570, 630, 610, 550] # 依次选择多个波段组合进行实, 组合的意思是[[850], [850, 870], [850, 870, 930], ..., [850, ..., 550]] # 每组波段下进行5折交叉验证 # 读取`split_64x64_1`中的`train/valid/test.txt`,按顺序划分为k折 class KFoldDataset(Dataset): def __init__(self, datapath, filelist, usedChannels): filelist = list(map(lambda x: os.path.join('/'.join(datapath.split('/')[:-1]), x.strip()), filelist)) self.samplelist = list(map(lambda x: [RecognizeDataset._load_image(x, 'Multi', usedChannels), getLabel(x)-1], filelist)) def __getitem__(self, index): image, label = self.samplelist[index] return image, label def __len__(self): return len(self.samplelist) CHANNEL_SORT = [850, 870, 930, 730, 790, 910, 770, 750, 670, 950, 990, 830, 890, 810, 970, 690, 710, 650, 590, 570, 630, 610, 550] usedChannelsList = [CHANNEL_SORT[:i+1] for i in range(23)] ## 读取所有文件 filelist = [] for mode in ['train', 'valid', 'test']: with open('./split/split_64x64_1/{}.txt'.format(mode), 'r') as f: filelist += f.readlines() ## 划分为k折 n_files_fold = len(filelist) // k foldlist = [] for i in range(k-1): foldlist += [filelist[i*n_files_fold: (i+1)*n_files_fold]] foldlist += [filelist[(k-1)*n_files_fold: ]] for i in range(k): ## k折交叉验证 validlist = foldlist[i] trainlist = list(filter(lambda x: x not in validlist, filelist)) for i_usedChannels in range(len(usedChannelsList)): usedChannels = usedChannelsList[i_usedChannels] print(getTime(), '[', i, '/', k, ']', len(usedChannels), '...') configer = EasyDict() configer.dsize = (64, 64) configer.datatype = 'Multi' configer.n_epoch = 300 configer.lrbase = 0.001 configer.n_channel = 23 configer.n_class = 63 configer.batchsize = 32 configer.stepsize = 250 configer.gamma = 0.2 configer.cuda = True configer.splitmode = 'split_{}x{}_1'.format(configer.dsize[0], configer.dsize[1]) configer.modelbase = 'recognize_vgg11_bn' configer.usedChannels = usedChannels configer.n_usedChannels = len(configer.usedChannels) configer.modelname = '{}_{}_{}_[{}_{}]fold'.\ format(configer.modelbase, configer.splitmode, '_'.join(list(map(str, configer.usedChannels))), i+1, k) configer.datapath = '/home/louishsu/Work/Workspace/ECUST2019_{}x{}'.\ format(configer.dsize[0], configer.dsize[1]) configer.logspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/logs/{}_{}_{}subjects_logs'.\ format(configer.modelbase, configer.splitmode, configer.n_class) configer.mdlspath = '/home/louishsu/Work/Workspace/HUAWEI/pytorch/modelfiles/{}_{}_{}subjects_models'.\ format(configer.modelbase, configer.splitmode, configer.n_class) ## datasets trainset = KFoldDataset(configer.datapath, trainlist, usedChannels) validset = KFoldDataset(configer.datapath, validlist, usedChannels) trainloader = DataLoader(trainset, configer.batchsize, shuffle=True) validloader = DataLoader(validset, configer.batchsize, shuffle=False) ## model modelpath = os.path.join(configer.mdlspath, configer.modelname) + '.pkl' modeldir = '/'.join(modelpath.split('/')[:-1]) if not os.path.exists(modeldir): os.makedirs(modeldir) model = modeldict[configer.modelbase](configer.n_usedChannels, configer.n_class, configer.dsize[0]) if configer.cuda and is_available(): model.cuda() ## loss loss = nn.CrossEntropyLoss() params = model.parameters() optimizer = optim.Adam(params, configer.lrbase, weight_decay=1e-3) scheduler = lr_scheduler.StepLR(optimizer, configer.stepsize, configer.gamma) logpath = os.path.join(configer.logspath, configer.modelname) if not os.path.exists(logpath): os.makedirs(logpath) logger = SummaryWriter(logpath) ## initialize acc_train = 0. acc_valid = 0. loss_train = float('inf') loss_valid = float('inf') loss_valid_last = float('inf') ## start training for i_epoch in range(configer.n_epoch): if configer.cuda and is_available(): empty_cache() scheduler.step(i_epoch) acc_train = []; acc_valid = [] loss_train = []; loss_valid = [] model.train() for i_batch, (X, y) in enumerate(trainloader): # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # backward optimizer.zero_grad() loss_i.backward() optimizer.step() loss_train += [loss_i.detach().cpu().numpy()] acc_train += [acc_i.cpu().numpy()] loss_train = np.mean(np.array(loss_train)) acc_train = np.mean(np.array(acc_train)) logger.add_scalar('accuracy', acc_train, i_epoch) logger.add_scalar('logloss', loss_train, i_epoch) logger.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) ## start testing model.eval() loss_test = [] acc_test = [] output = None for i_batch, (X, y) in enumerate(validloader): # get batch X = Variable(X.float()); y = Variable(y) if configer.cuda and is_available(): X = X.cuda(); y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # log loss_test += [loss_i.detach().cpu().numpy()] acc_test += [acc_i.cpu().numpy()] # save output if output is None: output = y_pred_prob.detach().cpu().numpy() else: output = np.concatenate([output, y_pred_prob.detach().cpu().numpy()], axis=0) # print('------------------------------------------------------------------------------------------------------------------') loss_test = np.mean(np.array(loss_test)) acc_test = np.mean(np.array(acc_test)) print_log = "{} || test | acc: {:2.2%}, loss: {:4.4f}".\ format(getTime(), acc_test, loss_test) print(print_log) with open(os.path.join(logpath, 'test_log.txt'), 'w') as f: f.write(print_log + '\n') np.save(os.path.join(logpath, 'test_out.npy'), output)
def train(configer): """ Update: 2019.04.24: 固定权值 """ ## datasets trainset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'train', configer.usedChannels) validset = RecognizeDataset(configer.datapath, configer.datatype, configer.splitmode, 'valid', configer.usedChannels) trainloader = DataLoader(trainset, configer.batchsize, shuffle=True) validloader = DataLoader(validset, configer.batchsize, shuffle=False) ## model: pre-initialized modelpath = os.path.join(configer.mdlspath, configer.modelname) + '.pkl' modeldir = '/'.join(modelpath.split('/')[:-1]) if not os.path.exists(modeldir): os.makedirs(modeldir) preInitdir = os.path.join('/'.join(configer.mdlspath.split('/')[:-1]), "preinit") if not os.path.exists(preInitdir): os.makedirs(preInitdir) preInitmodelpath = os.path.join(preInitdir, configer.modelbase + '.pkl') model = modeldict[configer.modelbase](configer.n_usedChannels, configer.n_class, configer.dsize[0]) if not os.path.exists(preInitmodelpath): model_state = model.state_dict() torch.save(model_state, preInitmodelpath) else: preinit_state = torch.load(preInitmodelpath) model_state = model.state_dict() toload_state = {k: v for k, v in preinit_state.items() \ if preinit_state[k].shape==model_state[k].shape} model_state.update(toload_state) model.load_state_dict(model_state) if configer.cuda and is_available(): model.cuda() ## loss loss = nn.CrossEntropyLoss() ## optimizer params = model.parameters() optimizer = optim.Adam(params, configer.lrbase, weight_decay=1e-3) ## learning rate scheduler scheduler = lr_scheduler.StepLR(optimizer, configer.stepsize, configer.gamma) ## log logpath = os.path.join(configer.logspath, configer.modelname) if not os.path.exists(logpath): os.makedirs(logpath) logger = SummaryWriter(logpath) ## initialize elapsed_time = 0 total_time = 0 start_time = 0 acc_train = 0. acc_valid = 0. loss_train = float('inf') loss_valid = float('inf') loss_valid_last = float('inf') ## start training for i_epoch in range(configer.n_epoch): if configer.cuda and is_available(): empty_cache() scheduler.step(i_epoch) acc_train = [] acc_valid = [] loss_train = [] loss_valid = [] model.train() start_time = time.time() for i_batch, (X, y) in enumerate(trainloader): # get batch X = Variable(X.float()) y = Variable(y) if configer.cuda and is_available(): X = X.cuda() y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # backward optimizer.zero_grad() loss_i.backward() optimizer.step() # time duration_time = time.time() - start_time start_time = time.time() elapsed_time += duration_time total_time = duration_time * configer.n_epoch * len( trainset) // configer.batchsize # log # print_log = "{} || Elapsed: {:.4f}h | Left: {:.4f}h | FPS: {:4.2f} || Epoch: [{:3d}]/[{:3d}] | Batch: [{:3d}]/[{:3d}] || lr: {:.6f} | accuracy: {:2.2%}, loss: {:4.4f}".\ # format(getTime(), elapsed_time/3600, (total_time - elapsed_time)/3600, configer.batchsize / duration_time, # i_epoch, configer.n_epoch, i_batch, len(trainset) // configer.batchsize, # scheduler.get_lr()[-1], acc_i, loss_i) # print(print_log) loss_train += [loss_i.detach().cpu().numpy()] acc_train += [acc_i.cpu().numpy()] # print('------------------------------------------------------------------------------------------------------------------') model.eval() for i_batch, (X, y) in enumerate(validloader): # get batch X = Variable(X.float()) y = Variable(y) if configer.cuda and is_available(): X = X.cuda() y = y.cuda() # forward y_pred_prob = model(X) loss_i = loss(y_pred_prob, y) acc_i = accuracy(y_pred_prob, y) # log # print_log = "{} || Epoch: [{:3d}]/[{:3d}] | Batch: [{:3d}]/[{:3d}] || accuracy: {:2.2%}, loss: {:4.4f}".\ # format(getTime(), i_epoch, configer.n_epoch, i_batch, len(validset) // configer.batchsize, acc_i, loss_i) # print(print_log) loss_valid += [loss_i.detach().cpu().numpy()] acc_valid += [acc_i.cpu().numpy()] # print('------------------------------------------------------------------------------------------------------------------') loss_train = np.mean(np.array(loss_train)) acc_train = np.mean(np.array(acc_train)) loss_valid = np.mean(np.array(loss_valid)) acc_valid = np.mean(np.array(acc_valid)) # print_log = "{} || Epoch: [{:3d}]/[{:3d}] || lr: {:.6f} || train | acc: {:2.2%}, loss: {:4.4f} || valid | acc: {:2.2%}, loss: {:4.4f}".\ # format(getTime(), i_epoch, configer.n_epoch, scheduler.get_lr()[-1], acc_train, loss_train, acc_valid, loss_valid) # print(print_log) logger.add_scalars('accuracy', { 'train': acc_train, 'valid': acc_valid }, i_epoch) logger.add_scalars('logloss', { 'train': loss_train, 'valid': loss_valid }, i_epoch) logger.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) # print('------------------------------------------------------------------------------------------------------------------') if loss_valid_last > loss_valid: loss_valid_last = loss_valid torch.save(model, modelpath)
def train(): learning_rate = configer.learningrate batch_size = configer.batchsize n_epoch = configer.n_epoch modelname = configer.modelname logger = init_logger() log_dir = os.path.join(configer.logspath, modelname) if not os.path.exists(log_dir): os.mkdir(log_dir) writer = SummaryWriter(log_dir) if configer.trainmode == 'Multi': trainsets = HyperECUST(configer.facesize, 'train') validsets = HyperECUST(configer.facesize, 'valid') elif configer.trainmode == 'RGB': trainsets = RGBECUST(configer.facesize, 'train') validsets = RGBECUST(configer.facesize, 'valid') trainloader = DataLoader(trainsets, batch_size, shuffle=True) validloader = DataLoader(validsets, batch_size) model, modelpath = init_model() # writer.add_graph(model, input_to_model=torch.Tensor(batch_size, configer.getint('global', 'N_CHANNLES'), # eval(configer.get('global', 'N_CHANNLES'))[0], eval(configer.get('global', 'N_CHANNLES'))[1])) print_log = 'load model: {}'.format(modelpath) print(print_log) logger.debug(print_log) loss = init_loss() optimizor = optim.Adam(model.parameters(), learning_rate) scheduler = lr_scheduler.StepLR(optimizor, configer.stepsize, configer.gamma) elapsed_time = 0 total_time = 0 acc_train_epoch = 0. acc_valid_epoch = 0. loss_train_epoch = float('inf') loss_valid_epoch = float('inf') loss_valid_epoch_last = loss_valid_epoch for i_epoch in range(n_epoch): scheduler.step(i_epoch) acc_train_epoch = [] acc_valid_epoch = [] loss_train_epoch = [] loss_valid_epoch = [] start_time = time.time() model.train() for i_batch, (X, y) in enumerate(trainloader): X = Variable(X.float()) y = Variable(y) if configer.cuda: X = X.cuda() y = y.cuda() y_pred_prob = model(X) loss_train_batch = loss(y_pred_prob, y) optimizor.zero_grad() loss_train_batch.backward() optimizor.step() duration_time = time.time() - start_time start_time = time.time() elapsed_time += duration_time total_time = duration_time * configer.n_epoch * len( trainsets) // configer.batchsize acc_train_batch = accuracy(y_pred_prob, y, multi=False) print_log = 'training... || Elapsed: {:.4f}h | Left: {:.4f}h | FPS: {:4.2f} || epoch [{:3d}]/[{:3d}] | batch [{:2d}]/[{:2d}] || accuracy: {:2.2%}, loss: {:4.4f}'.\ format(elapsed_time / 3600, (total_time - elapsed_time) / 3600, configer.batchsize / duration_time, i_epoch+1, n_epoch, i_batch+1, len(trainsets)//batch_size, acc_train_batch, loss_train_batch) print(print_log) logger.debug(print_log) acc_train_epoch.append(acc_train_batch.cpu().numpy()) loss_train_epoch.append(loss_train_batch.detach().cpu().numpy()) acc_train_epoch = np.mean(np.array(acc_train_epoch)) loss_train_epoch = np.mean(np.array(loss_train_epoch)) print_log = '-------------------------------------------------------------------------------------------------------------------------------------------------' model.eval() for i_batch, (X, y) in enumerate(validloader): X = Variable(X.float()) y = Variable(y) if configer.cuda: X = X.cuda() y = y.cuda() y_pred_prob = model(X) loss_valid_batch = loss(y_pred_prob, y) acc_valid_batch = accuracy(y_pred_prob, y, multi=False) print_log = 'validating... epoch [{:3d}]/[{:3d}] | batch [{:2d}]/[{:2d}] || accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_epoch+1, n_epoch, i_batch+1, len(validsets)//batch_size, acc_valid_batch, loss_valid_batch) print(print_log) logger.debug(print_log) acc_valid_epoch.append(acc_valid_batch.cpu().numpy()) loss_valid_epoch.append(loss_valid_batch.detach().cpu().numpy()) acc_valid_epoch = np.mean(np.array(acc_valid_epoch)) loss_valid_epoch = np.mean(np.array(loss_valid_epoch)) writer.add_scalars('accuracy', { 'train': acc_train_epoch, 'valid': acc_valid_epoch }, i_epoch) writer.add_scalars('logloss', { 'train': loss_train_epoch, 'valid': loss_valid_epoch }, i_epoch) writer.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) print_log = '-------------------------------------------------------------------------------------------------------------------------------------------------' print(print_log) logger.debug(print_log) print_log = 'epoch [{:3d}]/[{:3d}] || training: accuracy: {:2.2%}, loss: {:4.4f} | validing: accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_epoch, n_epoch, acc_train_epoch, loss_train_epoch, acc_valid_epoch, loss_valid_epoch) print(print_log) logger.debug(print_log) if loss_valid_epoch_last > loss_valid_epoch: torch.save(model, modelpath) acc_train_epoch_last = acc_train_epoch acc_valid_epoch_last = acc_valid_epoch loss_train_epoch_last = loss_train_epoch loss_valid_epoch_last = loss_valid_epoch print_log = 'model saved!' print(print_log) logger.debug(print_log) print_log = '=================================================================================================================================================' print(print_log) logger.debug(print_log)
def train(): learning_rate = configer.learningrate batch_size = configer.batchsize n_epoch = configer.n_epoch early_stopping = configer.earlystopping modelname = configer.modelname logger = init_logger() log_dir = os.path.join(configer.logspath, modelname) if not os.path.exists(log_dir): os.mkdir(log_dir) writer = SummaryWriter(log_dir) trainsets = HyperECUST(configer.splitmode, configer.facesize, 'train') trainloader = DataLoader(trainsets, batch_size, shuffle=True) validsets = HyperECUST(configer.splitmode, configer.facesize, 'valid') validloader = DataLoader(validsets, batch_size) model, modelpath = init_model() # writer.add_graph(model, input_to_model=torch.Tensor(batch_size, configer.getint('global', 'N_CHANNLES'), # eval(configer.get('global', 'N_CHANNLES'))[0], eval(configer.get('global', 'N_CHANNLES'))[1])) print_log = 'load model: {}'.format(modelpath) print(print_log); logger.debug(print_log) loss = init_loss() optimizor = optim.Adam(model.parameters(), learning_rate, betas=(0.9, 0.95), weight_decay=0.0005) scheduler = lr_scheduler.StepLR(optimizor, configer.stepsize, configer.gamma) acc_train_epoch = 0.; acc_valid_epoch = 0. loss_train_epoch = float('inf'); loss_valid_epoch = float('inf') acc_train_epoch_last = acc_train_epoch; acc_valid_epoch_last = acc_valid_epoch loss_train_epoch_last = loss_train_epoch; loss_valid_epoch_last = loss_valid_epoch for i_epoch in range(n_epoch): if torch.cuda.is_available(): torch.cuda.empty_cache() scheduler.step(i_epoch) acc_train_epoch = []; acc_valid_epoch = [] loss_train_epoch = []; loss_valid_epoch = [] model.train() for i_batch, (X, y) in enumerate(trainloader): X = Variable(X.float()) if torch.cuda.is_available(): X = X.cuda() y = y.cuda() y_pred_prob = model(X) loss_train_batch = loss(y_pred_prob, y) optimizor.zero_grad() loss_train_batch.backward() optimizor.step() acc_train_batch = accuracy(y_pred_prob, y, multi=False) print_log = 'training... epoch [{:3d}]/[{:3d}] | batch [{:2d}]/[{:2d}] || accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_epoch+1, n_epoch, i_batch+1, len(trainsets)//batch_size, acc_train_batch, loss_train_batch) print(print_log); logger.debug(print_log) acc_train_epoch.append(acc_train_batch.cpu().numpy()) loss_train_epoch.append(loss_train_batch.detach().cpu().numpy()) acc_train_epoch = np.mean(np.array(acc_train_epoch)) loss_train_epoch = np.mean(np.array(loss_train_epoch)) model.eval() for i_batch, (X, y) in enumerate(validloader): X = Variable(X.float()) if torch.cuda.is_available(): X = X.cuda() y = y.cuda() y_pred_prob = model(X) loss_valid_batch = loss(y_pred_prob, y) acc_valid_batch = accuracy(y_pred_prob, y, multi=False) print_log = 'validating... epoch [{:3d}]/[{:3d}] | batch [{:2d}]/[{:2d}] || accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_epoch+1, n_epoch, i_batch+1, len(validsets)//batch_size, acc_valid_batch, loss_valid_batch) print(print_log); logger.debug(print_log) acc_valid_epoch.append(acc_valid_batch.cpu().numpy()) loss_valid_epoch.append(loss_valid_batch.detach().cpu().numpy()) acc_valid_epoch = np.mean(np.array(acc_valid_epoch)) loss_valid_epoch = np.mean(np.array(loss_valid_epoch)) writer.add_scalars('accuracy', {'train': acc_train_epoch, 'valid': acc_valid_epoch}, i_epoch) writer.add_scalars('logloss', {'train': loss_train_epoch, 'valid': loss_valid_epoch}, i_epoch) writer.add_scalar('lr', scheduler.get_lr()[-1], i_epoch) print_log = '--------------------------------------------------------------------' print(print_log); logger.debug(print_log) print_log = 'epoch [{:3d}]/[{:3d}] || training: accuracy: {:2.2%}, loss: {:4.4f} | validing: accuracy: {:2.2%}, loss: {:4.4f}'.\ format(i_epoch, n_epoch, acc_train_epoch, loss_train_epoch, acc_valid_epoch, loss_valid_epoch) print(print_log); logger.debug(print_log) if early_stopping: if loss_valid_epoch_last > loss_valid_epoch: torch.save(model, modelpath) acc_train_epoch_last = acc_train_epoch; acc_valid_epoch_last = acc_valid_epoch loss_train_epoch_last = loss_train_epoch; loss_valid_epoch_last = loss_valid_epoch print_log = 'model saved!' print(print_log); logger.debug(print_log) else: torch.save(model, modelpath) acc_train_epoch_last = acc_train_epoch; acc_valid_epoch_last = acc_valid_epoch loss_train_epoch_last = loss_train_epoch; loss_valid_epoch_last = loss_valid_epoch print_log = 'model saved!' print(print_log); logger.debug(print_log) print_log = '====================================================================' print(print_log); logger.debug(print_log)