def main(): # Data Loader (Input Pipeline) print('loading dataset...') train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=False) # Define models print('building model...') cnn = CNN(input_channel=input_channel, n_outputs=num_classes) cnn.cuda() cnn.load_state_dict(torch.load(args.model)) # evaluate models with random weights test_acc = evaluate(test_loader, cnn) print( '=========> Test Accuracy on the %s test images: %.4f %% <===========' % (len(test_dataset), test_acc))
def train(args): assert args.num_classes common.make_dir(args.checkout_dir) nnet = CNN(args.left_context + args.right_context + 1, args.feat_dim, num_maps, pooling_size, filter_size, conn_dim, args.num_classes) print(nnet) nnet.cuda() criterion = nn.CrossEntropyLoss() optimizer = th.optim.Adam(nnet.parameters(), lr=args.learning_rate) train_dataset = THCHS30(root=args.data_dir, data_type='train', left_context=left_context, right_context=right_context, model_type='cnn') train_loader = data.DataLoader(dataset=train_dataset, batch_size=args.min_batch, shuffle=True, num_workers=6) test_dataset = THCHS30(root=args.data_dir, data_type='test', left_context=left_context, right_context=right_context, model_type='cnn') test_loader = data.DataLoader(dataset=test_dataset, batch_size=args.min_batch, shuffle=True, num_workers=6) cross_validate(-1, nnet, test_dataset, test_loader) for epoch in range(args.num_epochs): common.train_one_epoch(nnet, criterion, optimizer, train_loader) cross_validate(epoch, nnet, test_dataset, test_loader) th.save(nnet, common..join_path(args.checkout_dir, 'cnn.{}.pkl'.format(epoch + 1)))
def main(): # Data Loader (Input Pipeline) print 'loading dataset...' train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=True, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=True, shuffle=False) # Define models print 'building model...' cnn1 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn1.cuda() print cnn1.parameters optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn2.cuda() print cnn2.parameters optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) mean_pure_ratio1=0 mean_pure_ratio2=0 with open(txtfile, "a") as myfile: myfile.write('epoch: train_acc1 train_acc2 test_acc1 test_acc2 pure_ratio1 pure_ratio2\n') epoch=0 train_acc1=0 train_acc2=0 # evaluate models with random weights test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n") # training for epoch in range(1, args.n_epoch): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list=train(train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2) # evaluate models test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) # save results mean_pure_ratio1 = sum(pure_ratio_1_list)/len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list)/len(pure_ratio_2_list) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n")
def black_box_function(opt_param,ri): mean_pure_ratio1=0 mean_pure_ratio2=0 print('building model...') cnn1 = CNN(n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) # rate_schedule=opt_param[0]*(1-np.exp(-opt_param[2]*np.power(np.arange(args.n_epoch,dtype=float),opt_param[1])))+(1-opt_param[0])*(1-1/np.power((opt_param[4]*np.arange(args.n_epoch,dtype=float)+1),opt_param[3]))-np.power(np.arange(args.n_epoch,dtype=float)/args.n_epoch,opt_param[5])*opt_param[6] rate_schedule=opt_param[0]*(1-np.exp(-opt_param[2]*np.power(np.arange(args.n_epoch,dtype=float),opt_param[1])))\ +(1-opt_param[0])*opt_param[7]*(1-1/np.power((opt_param[4]*np.arange(args.n_epoch,dtype=float)+1),opt_param[3]))\ +(1-opt_param[0])*(1-opt_param[7])*(1-np.log(1+opt_param[8])/np.log(1+opt_param[8]+opt_param[9]*np.arange(args.n_epoch,dtype=float)))\ -np.power(np.arange(args.n_epoch,dtype=float)/args.n_epoch,opt_param[5])*opt_param[6]\ -np.log(1+np.power(np.arange(args.n_epoch,dtype=float),opt_param[11]))/np.log(1+np.power(args.n_epoch,opt_param[11]))*opt_param[10] print('Schedule:',rate_schedule,opt_param) epoch=0 train_acc1=0 train_acc2=0 # evaluate models with random weights test_acc1, test_acc2=evaluate(val_loader, cnn1, cnn2) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") # training for epoch in range(1, int(ri)): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list=train(train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, rate_schedule) # evaluate models test_acc1, test_acc2=evaluate(val_loader, cnn1, cnn2) # save results mean_pure_ratio1 = sum(pure_ratio_1_list)/len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list)/len(pure_ratio_2_list) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") return (test_acc1+test_acc2)/200
def train(dataset): model = CNN() # Choise GPU or CPU device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if device != "cpu": model.cuda(0) # Loss function for evaluate error criterion = nn.MSELoss() # Optimization method optimizer = optim.Adam(model.parameters(), lr=0.000001) loss_val = 0.0 loss_epoch = 100 index = 0 # Learning loop # By games for gi in range(dataset.train_games_count()): # By epoch for epoch in range(3): # By images in game for i in range(dataset.train_image_count(gi, BATCH)): # Parsed and load image,labels,steps from dataset image_batch, label_batch, steps_batch = dataset.get_train_batch(gi, i, BATCH) if device != "cpu": image_batch = image_batch.to(device) label_batch = label_batch.to(device) steps_batch = steps_batch.to(device) # Forward and learning optimizer.zero_grad() out = model(image_batch, steps_batch) loss = criterion(out, label_batch) loss.backward() optimizer.step() # Print loss index += 1 loss_val += loss if index % loss_epoch == 0: print("Loss", loss_val / loss_epoch) loss_val = 0.0 # Save model weights torch.save(model.state_dict(), os.path.join(model_save_path, model_name)) return
def train(X_train, y_train, X_val, y_val, X_test, y_test): X_train = torch.from_numpy(X_train).float() y_train = torch.from_numpy(y_train).long() X_test = torch.from_numpy(X_test).float() y_test = torch.from_numpy(y_test).long() model = CNN() # loss_fun = F.cross_entropy loss_func = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(model.parameters(), lr=0.001) if GPU: model = model.cuda() print('model:', model) print(X_train.shape) print(y_train.shape) trainset = TensorDataset(X_train, y_train) # valset = TensorDataset(torch.from_numpy(X_val), torch.from_numpy(y_val)) testset = TensorDataset(X_test, y_test) train_loader = DataLoaderX(trainset,batch_size=16,num_workers=0,pin_memory=True,shuffle=True) test_loader = DataLoaderX(testset,batch_size=16,num_workers=0,pin_memory=True) train_size = len(X_train) / 16 preloader = data_prefetcher(train_loader) for epoch in tqdm(range(EPOCH)): print('start epoch', epoch, ' / ', EPOCH) running_loss = 0.0 e_st = time.time() x, y = preloader for i, (x, y) in enumerate(tqdm(train_loader)): if i >= train_size - 1 : continue x = x.cuda() y = y.cuda() y_hat = model(x) loss = loss_func(y_hat, y).cuda() optimizer.zero_grad() loss.backward() optimizer.step() running_loss += loss.data.item() if i % 100 == 99: print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 100)) running_loss = 0.0 e_et = time.time() print('epoch:', epoch, ' use ', show_time(e_et-e_st)) print("Finished Training") print("Beginning Testing") correct = 0 total = 0 for data in test_loader: x, y = data y_hat = model(x.float()) _, pred = torch.max(y_hat.data, 1) total += y.size(0) correct += (pred == y).sum() print('Accuracy of model on test set:%d %%' % (100 * correct / total))
def trainer(X_train, y_train, X_val, y_val, X_test, y_test): model = CNN(batch_size=BATCH_SIZE) optimizer = torch.optim.Adam(model.parameters(), lr_) loss_func = torch.nn.functional.cross_entropy if torch.cuda.is_available(): model = model.cuda() loss_func = loss_func.cuda() print(model) model.train() tr_loss = 0 x_train, y_train = '', '' x_val, y_val = '', ''
def test(img_path, model_path, use_gpu = False): """!!! Useless !!! """ model = CNN() model.load(model_path) if use_gpu: model.cuda() char_table = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" # 把模型设为验证模式 from torchvision import transforms as T transforms = T.Compose([ T.Resize((128,128)), T.ToTensor(), ]) data = dataset.transforms(img_path, pre=False).unsqueeze(dim=0) model.eval() with torch.no_grad(): if use_gpu: data = data.cuda() score = model(data) score = decode(score) score = ''.join(map(lambda i: char_table[i], score[0])) return score
def test(dataset): model = CNN() # Load model weights model.load_state_dict(torch.load(os.path.join(model_save_path, model_name))) # Choise GPU or CPU device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if device != "cpu": model.cuda(0) acc_val = 0.0 acc_epoch = 100 index = 0 # Learning loop with torch.no_grad(): # By games for gi in range(dataset.train_games_count()): # By epoch for epoch in range(2): # By images in game for i in range(dataset.train_image_count(gi, BATCH)): # Parsed and load image,labels,steps from dataset image_batch, label_batch, steps_batch = dataset.get_test_batch( gi, i, BATCH) if device != "cpu": image_batch = image_batch.to(device) label_batch = label_batch.to(device) steps_batch = steps_batch.to(device) out = model(image_batch, steps_batch) # Print accurancy acc_val += 1 - F.mse_loss(out, label_batch) index += 1 if index % acc_epoch == 0: print("Accurancy", acc_val / acc_epoch) acc_val = 0.0 return
def main(): max_acc=0 num_param=12 opt_param=np.zeros(12) smax=int(np.floor(np.log(args.test_epoch)/np.log(args.eta))) B=(smax+1)*args.test_epoch for s in range(smax+1): s=smax-s n=int(np.ceil((smax+1)/(s+1)*np.power(args.eta,s))) r=args.test_epoch/np.power(args.eta,s) T=np.random.rand(n,num_param) T[:,2]*=0.5 T[:,4]*=0.5 T[:,9]*=0.5 T[:,5]/=0.5 T[:,6]*=0.5 T[:,11]/=0.5 T[:,10]*=0.5 for iii in range(s+1): ni=np.floor(n/np.power(args.eta,iii)) ri=np.ceil(r*np.power(args.eta,iii)) # maybe floor? test_runs=T.shape[0] L=np.zeros(test_runs) for jjj in range(test_runs): L[jjj]=black_box_function(T[jjj],ri) idx=np.argsort(L) T=T[idx[-int(np.floor(ni/args.eta)):]].copy() if L[-1]>max_acc: max_acc=L[-1] opt_param=T[-1].copy() mean_pure_ratio1=0 mean_pure_ratio2=0 rate_schedule=opt_param[0]*(1-np.exp(-opt_param[2]*np.power(np.arange(args.n_epoch,dtype=float),opt_param[1])))\ +(1-opt_param[0])*opt_param[7]*(1-1/np.power((opt_param[4]*np.arange(args.n_epoch,dtype=float)+1),opt_param[3]))\ +(1-opt_param[0])*(1-opt_param[7])*(1-np.log(1+opt_param[8])/np.log(1+opt_param[8]+opt_param[9]*np.arange(args.n_epoch,dtype=float)))\ -np.power(np.arange(args.n_epoch,dtype=float)/args.n_epoch,opt_param[5])*opt_param[6]\ -np.log(1+np.power(np.arange(args.n_epoch,dtype=float),opt_param[11]))/np.log(1+np.power(args.n_epoch,opt_param[11]))*opt_param[10] print('Schedule:',rate_schedule,opt_param) print('building model...') cnn1 = CNN(n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) train_acc1=0 train_acc2=0 # evaluate models with random weights test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") # training for epoch in range(1, args.n_epoch): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list=train(train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, rate_schedule) # evaluate models test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) # save results mean_pure_ratio1 = sum(pure_ratio_1_list)/len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list)/len(pure_ratio_2_list) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n")
name = 'VGGlike' elif args.nettype == 2: in_dim = (3, 32, 32) channels = 3 num_res_blocks = 4 net = ResidualNet(in_dim, channels, filters, num_res_blocks, dropout, nclasses) name = 'ResNet' else: print("Unknown net type") sys.exit() print(net) criterion = nn.CrossEntropyLoss() net.load_state_dict(torch.load(model_path)) print("Saved model loaded") '''Convert to cuda if available''' if torch.cuda.is_available() and cuda: print("CUDA is available, using GPU") print("Number of available devices: {}".format(torch.cuda.device_count())) print("Using device: {}".format(cuda_device)) torch.cuda.device(args.device) net.cuda() criterion = criterion.cuda() else: print("CUDA is NOT available, using CPU") '''Evaluate model''' print("Model performance on training set") evaluate(0, net, trainloader, criterion, cuda, batch_size) print("Model performance on validation set") evaluate(0, net, testloader, criterion, cuda, batch_size)
def main(parser): # prepare parameters opt = parser.parse_args() BATCHSIZE = opt.batch_size EPOCHS = opt.epochs resnet_path = opt.resnet_path N_ATT = opt.n_att ##load data transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.Resize(opt.SIZE_TRAIN), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) transform_test = transforms.Compose([ transforms.Resize(opt.SIZE_TEST), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) trainset = CelebA(opt, transform=transform_train, tag=opt.tag, pg_tag=opt.pg_tag) train_loader = DataLoader( dataset=trainset, batch_size=BATCHSIZE, num_workers=8, pin_memory=True, ) testset = CelebA(opt, transform=transform_test, setting='test', tag=opt.tag, pg_tag=opt.pg_tag) test_loader = DataLoader( dataset=testset, batch_size=BATCHSIZE, num_workers=8, pin_memory=True, ) n_train = len(trainset) # build model print('init networks') cnn_model = CNN(num_layers=opt.net_depth, drop_ratio=opt.drop_ratio, mode=opt.net_mode, resnet_path=resnet_path) attribute_model = AttributeNetwork(40, 256, 512) sam = SAM(input_size=512, output_size=40, n_att=N_ATT) arcmargin_compute = DCM_Logit(d=opt.d, m=opt.margin) cnn_model.cuda() attribute_model.cuda() sam.cuda() # multi-gpu setting if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs to train our model !!!") cnn_model = nn.DataParallel(cnn_model) attribute_model = nn.DataParallel(attribute_model) sam = nn.DataParallel(sam) # model optimizers cnn_model_optim = torch.optim.Adam(cnn_model.parameters(), lr=opt.lr_cnn, weight_decay=opt.weight_decay) attribute_model_optim = torch.optim.Adam(attribute_model.parameters(), lr=opt.lr_attribute, weight_decay=opt.weight_decay) sam_optim = torch.optim.Adam(sam.parameters(), lr=opt.lr_sam, weight_decay=opt.weight_decay) print("training...") m = 0 loss_list = [] dcm_loss_list = [] cea_loss_list = [] test_i2a_average_acc_list = [] test_a2i_average_acc_list = [] test_att_classify_average_acc_list = [] iter_list = [] for epoch in tqdm(range(EPOCHS)): cnn_model.train() sam.train() attribute_model.train() i = 0 epoch_start_time = datetime.now() for batch_imgs, batch_atts, batch_binary_atts, batch_labels in train_loader: # prepare data unique_labels = torch.unique(batch_labels).numpy() unique_batch_atts = torch.from_numpy( trainset.class_embeddings[unique_labels, :]) batch_imgs = Variable(batch_imgs).cuda().float() # 32*1024 batch_binary_atts = Variable( batch_binary_atts).cuda().float() # 32*1024 unique_batch_atts = Variable(unique_batch_atts).cuda().float() # get model output fea_maps, feas_vec = cnn_model(batch_imgs) unique_attribute_network_out = attribute_model(unique_batch_atts) pred_atts, batch_feas, spatial_fea_maps = sam(fea_maps) # compute loss re_batch_labels = [] for label in batch_labels.numpy(): idx = np.where(unique_labels == label)[0][0] re_batch_labels.append(idx) re_batch_labels = torch.LongTensor(re_batch_labels).cuda() output = arcmargin_compute.forward( input=batch_feas, attribute_output=unique_attribute_network_out, label=re_batch_labels) target = Variable(re_batch_labels).cuda() CEA = nn.BCELoss().cuda() DCM = nn.CrossEntropyLoss().cuda() cea_loss = CEA(input=pred_atts, target=batch_binary_atts) * 15 dcm_loss = DCM.forward(input=output, target=target) loss = cea_loss + dcm_loss # update cnn_model.zero_grad() attribute_model.zero_grad() sam.zero_grad() loss.backward() cnn_model_optim.step() attribute_model_optim.step() sam_optim.step() if m == 0: iter_start_time = datetime.now() loss_list.append(loss.cpu().detach().numpy()) dcm_loss_list.append(dcm_loss.cpu().detach().numpy()) cea_loss_list.append(cea_loss.cpu().detach().numpy()) iter_list.append(m) print('loss:{:.5},dcm_loss:{:.5},cea_loss:{:.5}'.format( loss.cpu().detach().numpy(), dcm_loss.cpu().detach().numpy(), cea_loss.cpu().detach().numpy())) if (m != 0) and (m % 500 == 0): iter_end_time = datetime.now() iter_spent_time = (iter_end_time - iter_start_time).seconds / 60 iter_start_time = iter_end_time loss_list.append(loss.cpu().detach().numpy()) dcm_loss_list.append(dcm_loss.cpu().detach().numpy()) cea_loss_list.append(cea_loss.cpu().detach().numpy()) iter_list.append(m) print( 'Have finished {} samples and remain {} samples, spend {:.2f} minutes' .format(BATCHSIZE * 500, n_train - BATCHSIZE * i, iter_spent_time)) print('loss:{:.5},dcm_loss:{:.5},cea_loss:{:.5}'.format( loss.cpu().detach().numpy(), dcm_loss.cpu().detach().numpy(), cea_loss.cpu().detach().numpy(), )) m += 1 i += 1 epoch_end_time = datetime.now() epoch_spend_time = (epoch_end_time - epoch_start_time).seconds / 60 print('Spend {:.2f} minutes to complete one epoch '.format( epoch_spend_time)) print('#############') print('Testing ...') test_i2a_average_acc, test_a2i_average_acc,\ test_att_classify_average_acc = compute_topx_acc(cnn_model=cnn_model, attribute_model=attribute_model, sam=sam, test_loader=test_loader, testset=testset, ) test_i2a_average_acc_list.append(test_i2a_average_acc) test_a2i_average_acc_list.append(test_a2i_average_acc) test_att_classify_average_acc_list.append( test_att_classify_average_acc) for q, top_number in enumerate(opt.top_x): print( 'Testset Imgae to Attribute top-{} average class accuracy:{:.2f}' .format(top_number, test_i2a_average_acc[q])) print('\n') for q, top_number in enumerate(opt.top_x): print( 'Testset Attribute to Image top-{} average class accuracy:{:.2f}' .format(top_number, test_a2i_average_acc[q])) print('Testset att classify acc is {:.2f}'.format( test_att_classify_average_acc)) if not os.path.exists(opt.basemodel_dir): os.makedirs(opt.basemodel_dir) basemodel_dir = opt.basemodel_dir + '/' + 'Top_' + str( opt.n_att) + '_spatial_model' if not os.path.exists(basemodel_dir): os.makedirs(basemodel_dir) if not os.path.exists(basemodel_dir + '/' + opt.cnn_model_dir): os.makedirs(basemodel_dir + '/' + opt.cnn_model_dir) if not os.path.exists(basemodel_dir + '/' + opt.att_model_dir): os.makedirs(basemodel_dir + '/' + opt.att_model_dir) if not os.path.exists(basemodel_dir + '/' + opt.sam_model_dir): os.makedirs(basemodel_dir + '/' + opt.sam_model_dir) print('#############') print('save networks') cnn_model_name = opt.cnn_model_name + '_epoch_' + str(epoch) + '.pkl' attribute_model_name = opt.attribute_model_name + '_epoch_' + str( epoch) + '.pkl' sam_model_name = opt.sam_model_name + '_epoch_' + str(epoch) + '.pkl' if torch.cuda.device_count() > 1: torch.save( cnn_model.module.state_dict(), basemodel_dir + '/' + opt.cnn_model_dir + '/' + cnn_model_name) torch.save( attribute_model.module.state_dict(), basemodel_dir + '/' + opt.att_model_dir + '/' + attribute_model_name) torch.save( sam.module.state_dict(), basemodel_dir + '/' + opt.sam_model_dir + '/' + sam_model_name) else: torch.save( cnn_model.state_dict(), basemodel_dir + '/' + opt.cnn_model_dir + '/' + cnn_model_name) torch.save( attribute_model.state_dict(), basemodel_dir + '/' + opt.att_model_dir + '/' + attribute_model_name) torch.save( sam.state_dict(), basemodel_dir + '/' + opt.sam_model_dir + '/' + sam_model_name) loss_arr = np.array(loss_list) dcm_loss_arr = np.array(dcm_loss_list) cea_loss_arr = np.array(cea_loss_list) iter_arr = np.array(iter_list) test_i2a_average_acc_arr = np.array(test_i2a_average_acc_list) test_a2i_average_acc_arr = np.array(test_a2i_average_acc_list) test_att_classify_average_acc_arr = np.array( test_att_classify_average_acc_list) results_name = basemodel_dir + '/' + 'Top_' + str( N_ATT) + '_spatial_results.mat' scio.savemat( results_name, { 'top-number': np.array(opt.top_x), 'loss': loss_arr, 'dcm_loss': dcm_loss_arr, 'cea_loss': cea_loss_arr, 'iter': iter_arr, 'test_i2a_average_acc': test_i2a_average_acc_arr, 'test_a2i_average_acc': test_a2i_average_acc_arr, 'test_att_classify_average_acc': test_att_classify_average_acc_arr, 'drop_ratio': opt.drop_ratio, 'weight_decay': opt.weight_decay, 'margin': opt.margin, 'd': opt.d, 'lr_cnn': opt.lr_cnn, 'lr_attribute': opt.lr_attribute, 'lr_sam': opt.lr_sam, 'batch_size': opt.batch_size, 'attribute_hidden_size': np.array([256]), })
def main(args): num_epochs = args.ne batch_size = args.bs transform = transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomHorizontalFlip(), transforms.RandomCrop(227), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) dataset_path = args.dataset_path json_labels_path = args.json_labels_path data_loader = get_wiki_data_loader(dataset_path, json_labels_path, transform, batch_size, shuffle=True, num_workers=args.n_workers) if args.out_dim == -1: out_dim = None else: out_dim = args.out_dim if args.ttn: cnn = CNN(args.n_topics, args.n_kernels, mixture_model=False, cnn=args.cnn, pretrained=args.pretrained) else: cnn = CNN(args.n_topics, args.n_kernels, out_dim=out_dim, cnn=args.cnn, pretrained=args.pretrained) if torch.cuda.is_available(): cnn.cuda() cnn.train() optimizer = optim.SGD(cnn.parameters(), lr=args.lr, momentum=args.mm) # optimizer = optim.Adam(cnn.parameters()) # optimizer = optim.RMSprop(cnn.parameters(), lr= args.lr, momentum=args.mm) exp = Experiment(args, args.exp_name) if args.ttn: loss_fn = torch.nn.modules.loss.BCEWithLogitsLoss(reduction='sum') else: loss_fn = nll_loss learning_rate = args.lr losses = [] for epoch in range(num_epochs): learning_rate = update_lr_epoch(epoch, args, learning_rate, optimizer) for step, (images, ts) in enumerate(data_loader): if torch.cuda.is_available(): images = images.cuda() ts = ts.cuda() cnn.zero_grad() if not args.ttn: alpha, sigma, mu = cnn(images) loss = loss_fn(alpha, sigma, mu, ts) else: out = cnn(images) loss = loss_fn(out, ts) loss.backward() if args.clipping != 0.: torch.nn.utils.clip_grad_norm_(cnn.parameters(), args.clipping) optimizer.step() losses.append(float(loss)) exp.save_loss(epoch, step, loss) print('Epoch ' + str(epoch + 1) + '/' + str(num_epochs) + ' - Step ' + str(step + 1) + '/' + str(len(data_loader)) + ' - Loss: ' + str(float(loss))) exp.save_loss_epoch(epoch, losses) losses = [] if epoch % args.save_epoch == 0 and epoch > 0: exp.save_model(epoch, cnn) exp.save_model('last', cnn)
def train_test(Q, x_train, x_test, y_train, y_test, batch_size): train_loader, test_loader = create_train_test_loaders( Q, x_train, x_test, y_train, y_test, batch_size) cnn = CNN(input_size=num_filters, hidden_size=hidden_size, num_classes=np.unique(y).size, dim=dim, num_kernels=num_kernels, max_document_length=max_document_length) if torch.cuda.is_available(): cnn.cuda() if torch.cuda.is_available(): criterion = nn.CrossEntropyLoss().cuda() else: criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) for epoch in range(num_epochs): for i, (graphs, labels) in enumerate(train_loader): graphs = Variable(graphs) labels = Variable(labels) optimizer.zero_grad() outputs = cnn(graphs) if torch.cuda.is_available(): loss = criterion(outputs, labels.cuda()) else: loss = criterion(outputs, labels) loss.backward() optimizer.step() # Test the Model cnn.eval() correct = 0 total = 0 TP = 0 TN = 0 FP = 0 FN = 0 predict = [] label = [] output = [] for graphs, labels in test_loader: graphs = Variable(graphs) outputs = cnn(graphs) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels.cuda()).sum() TP += (predicted + labels.cuda() == 2).sum() FP += (predicted * 5 + labels.cuda() * 1 == 5).sum() FN += (predicted * 1 + labels.cuda() * 5 == 5).sum() TN += (predicted + labels.cuda() == 0).sum() predict.append(predicted) label.append(labels) output.append(outputs.data) if TP + FP == 0: precision = 0 else: precision = TP / (TP + FP) if TP + FN == 0: recall = 0 else: recall = TP / (TP + FN) l = np.zeros((len(label))) for i in range(len(label)): l[i] = int(label[i]) s = np.zeros((len(output))) for i in range(len(output)): s[i] = output[i][0][1] return TP, TN, FP, FN, precision, recall, l, s
train_accs = [] test_accs = [] it = 0 print("Starting cross-validation...") for train_index, test_index in kf.split(x): it += 1 x_train, x_test = x[train_index], x[test_index] y_train, y_test = y[train_index], y[test_index] train_loader, test_loader = create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size) cnn = CNN(input_size=num_filters, hidden_size=hidden_size, num_classes=np.unique(y).size, dim=dim, num_kernels=num_kernels, max_document_length=max_document_length) if torch.cuda.is_available(): cnn.cuda() # Loss and Optimizer if torch.cuda.is_available(): criterion = nn.CrossEntropyLoss().cuda() else: criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) # Train the Model for epoch in range(num_epochs): total_loss = 0 for i, (graphs, labels) in enumerate(train_loader): graphs = Variable(graphs).cuda() labels = Variable(labels).cuda() optimizer.zero_grad()
def black_box_function(opt_param): mean_pure_ratio1 = 0 mean_pure_ratio2 = 0 print('building model...') cnn1 = CNN(n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) rate_schedule = opt_param.copy() print('Schedule:', rate_schedule) epoch = 0 train_acc1 = 0 train_acc2 = 0 # evaluate models with random weights test_acc1, test_acc2 = evaluate(test_loader, cnn1, cnn2) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(train_acc2) + ' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") # training for epoch in range(1, args.n_epoch): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list = train( train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, rate_schedule) # evaluate models test_acc1, test_acc2 = evaluate(test_loader, cnn1, cnn2) # save results mean_pure_ratio1 = sum(pure_ratio_1_list) / len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list) / len(pure_ratio_2_list) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(train_acc2) + ' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") return (test_acc1 + test_acc2) / 200
def main(): global args, best_prec1 args = parser.parse_args() with open(args.config) as f: config = yaml.load(f) for key in config: for k, v in config[key].items(): setattr(args, k, v) print('Enabled distributed training.') rank, world_size = init_dist(backend='nccl', port=args.port) args.rank = rank args.world_size = world_size model_dir = args.model_dir if args.rank == 0 and not os.path.exists(model_dir): os.makedirs(model_dir) if args.evaluate: args.resume = True model = CNN(args) model.cuda() broadcast_params(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.base_lr, momentum=args.momentum, weight_decay=args.weight_decay) # auto resume from a checkpoint start_epoch = 0 if args.evaluate: load_state_ckpt(args.checkpoint_path, model) else: best_prec1, start_epoch = load_state(model_dir, model, optimizer=optimizer) if args.rank == 0: writer = SummaryWriter(model_dir) else: writer = None cudnn.benchmark = True normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = ImagenetDataset( args.train_root, args.train_source, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ColorAugmentation(), normalize, ])) val_dataset = ImagenetDataset( args.val_root, args.val_source, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) train_sampler = DistributedSampler(train_dataset) val_sampler = DistributedSampler(val_dataset) train_loader = DataLoader(train_dataset, batch_size=args.batch_size // args.world_size, shuffle=False, num_workers=args.workers, pin_memory=False, sampler=train_sampler) val_loader = DataLoader(val_dataset, batch_size=args.batch_size // args.world_size, shuffle=False, num_workers=args.workers, pin_memory=False, sampler=val_sampler) if args.evaluate: validate(val_loader, model, criterion, 0, writer) return niters = len(train_loader) lr_scheduler = LRScheduler(optimizer, niters, args) for epoch in range(start_epoch, args.epochs): train_sampler.set_epoch(epoch) # train for one epoch train(train_loader, model, criterion, optimizer, lr_scheduler, epoch, writer) # evaluate on validation set prec1 = validate(val_loader, model, criterion, epoch, writer) if rank == 0: # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( model_dir, { 'epoch': epoch + 1, 'model': args.config.rsplit('/', 1)[-1].split('.yaml')[0], 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best)
def main(): np.random.seed(args.seed) cur_acc = 0 max_acc = 0 num_param = 3 cur_param = np.zeros(num_param) max_pt = np.zeros(num_param) for iii in range(args.n_iter): for jjj in range(args.n_samples): for kkk in range(num_param): cur_param[kkk] = np.random.beta(1, 1) cur_param[1] *= args.n_epoch cur_param[2] /= 0.5 cur_acc = black_box_function(cur_param) if max_acc < cur_acc: max_acc = cur_acc max_pt = cur_param.copy() hyp_param = np.zeros(6) hyp_param[0] = max_pt[0] hyp_param[1] = 1 - max_pt[0] hyp_param[2] = max_pt[1] hyp_param[3] = max_pt[2] hyp_param[4] = max_pt[3] hyp_param[5] = max_pt[4] rate_schedule = hyp_param[0] * (1 - np.exp(-hyp_param[3] * np.power( np.arange(args.n_epoch, dtype=float), hyp_param[2]))) + hyp_param[1] * (1 - 1 / np.power( (hyp_param[5] * np.arange(args.n_epoch, dtype=float) + 1), hyp_param[4])) print('Schedule:', rate_schedule, hyp_param) ''' rate_schedule=np.ones(args.n_epoch)*forget_rate rate_schedule[:10]=np.arange(10,dtype=float)/10*forget_rate rate_schedule[10:]=np.arange(args.n_epoch-10,dtype=float)/(args.n_epoch-10)*forget_rate+forget_rate rate_schedule=np.zeros(args.n_epoch) hyp_param=np.asarray([0.24419,0.75581,1,0.00135,0.68079,0.03778]) rate_schedule=hyp_param[0]*(1-np.exp(-hyp_param[3]*np.power(np.arange(args.n_epoch,dtype=float),hyp_param[2])))+hyp_param[1]*(1-1/np.power((hyp_param[5]*np.arange(args.n_epoch,dtype=float)+1),hyp_param[4])) print(rate_schedule) ''' epoch = 0 mean_pure_ratio1 = 0 mean_pure_ratio2 = 0 print('building model...') cnn1 = CNN(n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) train_acc1 = 0 train_acc2 = 0 # evaluate models with random weights test_acc1, test_acc2 = evaluate(test_loader, cnn1, cnn2) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(train_acc2) + ' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n") # training for epoch in range(1, args.n_epoch): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list = train( train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, rate_schedule) # evaluate models test_acc1, test_acc2 = evaluate(test_loader, cnn1, cnn2) # save results mean_pure_ratio1 = sum(pure_ratio_1_list) / len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list) / len(pure_ratio_2_list) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(train_acc2) + ' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + ' ' + str(rate_schedule[epoch]) + "\n")
def train(X_train, y_train, X_val, y_val, X_test, y_test): X_train = torch.from_numpy(X_train).float() y_train = torch.from_numpy(y_train).long() X_test = torch.from_numpy(X_test).float() y_test = torch.from_numpy(y_test).long() # torch.distributed.init_process_group(backend="nccl") model = CNN(batch_size=BATCH_SIZE) # loss_func = F.cross_entropy # loss_func = loss_func. loss_func = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) if GPU: # model = DistributedDataParallel(model) # model = nn.DataParallel(model) model = model.cuda() print('model:', model) print(X_train.shape) print(y_train.shape) trainset = TensorDataset(X_train, y_train) # valset = TensorDataset(torch.from_numpy(X_val), torch.from_numpy(y_val)) testset = TensorDataset(X_test, y_test) test_loader = DataLoaderX(testset, batch_size=BATCH_SIZE, num_workers=32, pin_memory=True) # train_size = len(X_train) / 16 train_loader = DataLoaderX(trainset, batch_size=BATCH_SIZE, num_workers=8, pin_memory=True, shuffle=True, drop_last=True) for epoch in tqdm(range(EPOCH)): print('start epoch', epoch, ' / ', EPOCH) running_loss = 0.0 e_st = time.time() for i, (x, y) in enumerate(tqdm(train_loader)): # if i >= train_size - 1 : # continue x = x.cuda() y = y.cuda() y_hat = model(x) # print('y',y) # print('y_hat', y) loss = loss_func(y_hat, y).cuda() # print('loss:',loss) optimizer.zero_grad() # print("y", y.shape) # print("y_h", y_hat.shape) # y_pre = torch.argmax(y_hat, dim=1).cuda() # y_pre = torch.tensor(y_pre, dtype=torch.float) # print("ypre:", y_pre.type()) # print("yhat:", y_hat.type()) # print("y:", y.type()) loss.backward() optimizer.step() running_loss += loss.item() if i % 100 == 99: print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 100)) running_loss = 0.0 e_et = time.time() print('\nepoch:', epoch, ' use ', show_time(e_et - e_st)) print("Finished Training") print("Beginning Testing") correct = 0 total = 0 for data in test_loader: x, y = data x = x.cuda() y = y.cuda() print('y', y) y_hat = model(x.float()) print('y_hat', y_hat) _, pred = torch.max(y_hat.data, 1) print('pred:', pred) total += y.size(0) correct += (pred == y).sum() print('Accuracy of model on test set:%d %%' % (100 * correct // total))
def main(): # Data Loader (Input Pipeline) print('loading dataset...') train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=True, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=True, shuffle=False) # Define models print('building model...') cnn1 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) mean_pure_ratio1=0 mean_pure_ratio2=0 with open(txtfile, "a") as myfile: myfile.write('epoch: train_acc1 train_acc2 test_acc1 test_acc2 pure_ratio1 pure_ratio2\n') epoch=0 train_acc1=0 train_acc2=0 global flag flag = 0 #flag is a parameter of loss_coteaching and indicates when to start using the statistical model global N1 global N2 global mean1 global mean2 global cov1 global cov2 # evaluate models with random weights test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n") # training for epoch in range(1, args.n_epoch): if epoch % 50 == 0 and epoch > 0: time.sleep(300) #let GPU cool for 5mins if epoch >= args.use_model: flag = 1 # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list=train(train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, flag) # evaluate models test_acc1, test_acc2=evaluate(test_loader, cnn1, cnn2) #clear cuda torch.cuda.empty_cache() # save results mean_pure_ratio1 = sum(pure_ratio_1_list)/len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list)/len(pure_ratio_2_list) print('Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %%' % (epoch+1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, mean_pure_ratio1, mean_pure_ratio2)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n")
def train(): train_data_loader, test_data_loader = load_data() cnn = CNN() if torch.cuda.is_available(): cnn.cuda() if restor: cnn.load_state_dict(torch.load(MODEL_PATH)) optimizer = torch.optim.Adam(cnn.parameters(), lr=base_lr) # optimize criterion = nn.MultiLabelSoftMarginLoss() train_acc_epoch = [] test_acc_epoch = [] loss_epoch = [] for epoch in range(max_epoch): start = time.time() cnn.train() acc_history = [] loss_history = [] # in train set for img, target in train_data_loader: img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) loss = criterion(output, target) optimizer.zero_grad() loss.backward() optimizer.step() acc_history.append(calculat_acc(output, target)) loss_history.append(float(loss)) train_acc, loss = cal_acc_loss(acc_history, loss_history) loss_history = [] acc_history = [] cnn.eval() # in test set for img, target in test_data_loader: img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) acc_history.append(calculat_acc(output, target)) test_acc, nothing = cal_acc_loss(acc_history, loss_history) print('train loss:{:.5f}'.format(loss)) print('train acc:{:.5f}'.format(train_acc)) print('test acc: {:.5f}'.format(test_acc)) print('epoch: {} , using time: {:.5}\n'.format(epoch + 1, time.time() - start)) train_acc_epoch.append(train_acc) test_acc_epoch.append(test_acc) loss_epoch.append(loss) torch.save(cnn.state_dict(), MODEL_PATH) if loss < eps: break return train_acc_epoch, test_acc_epoch, loss_epoch
def train(): transforms_train = Compose( [ToTensor(), RandomAffine(10, translate=(0.02, 0.05))]) train_dataset = CaptchaData('./set-train', transform=transforms_train) train_data_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=0, shuffle=True, drop_last=True) # train_data_loader.dataset.set_use_cache(use_cache=True) test_data = CaptchaData('./set-test', transform=None) test_data_loader = DataLoader(test_data, batch_size=batch_size, num_workers=0, shuffle=True, drop_last=True) # test_data_loader.dataset.set_use_cache(use_cache=True) print('train set', len(train_dataset)) print('test set', len(test_data)) cnn = CNN() if torch.cuda.is_available(): cnn.cuda() if restor: cnn.load_state_dict(torch.load(model_path)) # freezing_layers = list(cnn.named_parameters())[:10] # for param in freezing_layers: # param[1].requires_grad = False # print('freezing layer:', param[0]) optimizer = torch.optim.AdamW(cnn.parameters(), lr=base_lr, weight_decay=True) criterion = nn.MultiLabelSoftMarginLoss() sche = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=2, threshold=.002, verbose=True) for epoch in range(max_epoch): start_ = time.time() loss_history = [] acc_history = [] cnn.train() for img, target in (train_data_loader): # pdb.set_trace() img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) loss = criterion(output, target) optimizer.zero_grad() loss.backward() optimizer.step() acc = calculat_acc(output, target) acc_history.append(float(acc)) loss_history.append(float(loss)) print('train_loss: {:.4}|train_acc: {:.4}'.format( torch.mean(torch.Tensor(loss_history)), torch.mean(torch.Tensor(acc_history)), )) loss_history = [] acc_history = [] cnn.eval() with torch.no_grad(): for img, target in (test_data_loader): img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) loss = criterion(output, target) acc = calculat_acc(output, target) acc_history.append(float(acc)) loss_history.append(float(loss)) test_loss = torch.mean(torch.Tensor(loss_history)) print('test_loss: {:.4}|test_acc: {:.4}'.format( test_loss, torch.mean(torch.Tensor(acc_history)), )) print('epoch: {}|time: {:.4f}'.format(epoch, time.time() - start_)) torch.save(cnn.state_dict(), model_path % epoch) sche.step(test_loss)
print(test_2_list) data1 = myData(datapath, train_list) data2 = myData(datapath, test_2_list) train_data = DataLoader(dataset=data1, batch_size=48, shuffle=True) test_data = DataLoader(dataset=data2, batch_size=1, shuffle=True) data = {'train': train_data, 'test': test_data} data_len = {'train': len(train_list), 'test': len(test_2_list)} cnn = CNN() if torch.cuda.is_available(): cnn = cnn.cuda() print('正在使用GPU...') else: print('正在使用CPU...') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") cnn.apply(weights_init) optimizer = torch.optim.Adam(cnn.parameters(), lr=lr) loss_func = nn.CrossEntropyLoss() train_model(cnn, num_epochs, data, device, optimizer, loss_func, data_len, savepath)
def train(path=None, log_path=None): """Train the CNN mode. Args: path (str): checkpoint file path. log_path (str): log_path. default='./log/train_<datetime>.log' """ """ ===== Constant var. start =====""" train_comment = "" use_gpu = True num_workers = 7 batch_size = 128 lr = 0.001 lr_decay = 0.9 max_epoch = 500 stat_freq = 10 model_name = "test" """ ===== Constant var. end =====""" # step0: init. log and checkpoint dir. checkpoints_dir = "./checkpoints/" + model_name if len(train_comment) > 0: checkpoints_dir = checkpoints_dir + "_" + train_comment if not os.path.isdir("./checkpoints"): os.mkdir("./checkpoints") if not os.path.isdir(checkpoints_dir): os.mkdir(checkpoints_dir) if log_path == None: if not os.path.isdir("./log"): os.mkdir("./log") if not os.path.exists('./log/' + model_name): os.makedirs("./log/" + model_name) log_path = "./log/{}".format(model_name) # step1: dataset val_data = Data(train=False) val_dataloader = DataLoader(val_data, 100, num_workers=num_workers) train_data = Data(train=True) train_dataloader = DataLoader(train_data, batch_size, shuffle=True, num_workers=num_workers, pin_memory=True) writer = SummaryWriter(log_path) best_acc_img = 0 with open(log_path + "/log.txt", "w") as log_file: # step2: instance and load model model = CNN() if path != None: print('using mode "{}"'.format(path)) print('using mode "{}"'.format(path), file=log_file, flush=True) model.load(path) else: print("init model by orthogonal_", file=log_file, flush=True) for name, param in model.named_parameters(): if len(param.shape) > 1: torch.nn.init.orthogonal_(param) if use_gpu: model.cuda() # summary(model, (1, 128, 128)) # step3: loss function and optimizer criterion = loss_ optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) previous_loss = 1e100 # epoch loop for epoch in range(max_epoch): running_loss = 0.0 total_loss = [] # batch loop pbar = tqdm(enumerate(train_dataloader)) for i, (data, label) in pbar: input = data target = label if use_gpu: input = input.cuda() target = target.cuda() optimizer.zero_grad() score = model(input) loss = criterion(score, target) loss.backward() optimizer.step() running_loss += loss.item() total_loss.append(loss.item()) if (i + 1) % stat_freq == 0: pbar.set_description( "[%d, %5d] loss: %.3f" % (epoch + 1, i + 1, running_loss / stat_freq)) writer.add_scalar('train/loss', running_loss / stat_freq, epoch * len(train_dataloader) + i) running_loss = 0.0 previous_loss = np.mean(total_loss) acc_img, acc_digit, im_show = val(model, val_dataloader, use_gpu) writer.add_scalar('eval/acc_img', acc_img, epoch * len(train_dataloader)) writer.add_scalar('eval/acc_digit', acc_digit, epoch * len(train_dataloader)) im_show[0] = cv2.putText(im_show[0], ''.join(Data.decode(im_show[1])), (20, 20), 2, 1, (255, 0, 255)) writer.add_image("img", torch.tensor(np.transpose(im_show[0], (2, 0, 1))), epoch * len(train_dataloader)) if acc_img > best_acc_img: save_path = "{}/model.pth".format(checkpoints_dir) model.save(save_path) print("acc_img : {}, acc_digit : {}, loss : {}".format( acc_img, acc_digit, previous_loss)) if np.mean(total_loss) > previous_loss: lr = lr * lr_decay print("reduce loss from to {}".format(lr))
perf_dict = {} for emb in word_embeddings: print ("Running model for embedding" + str(emb)) emb_dim = int(emb.split('_')[2].split('s')[1]) amazon = Amazon_loader(emb_file=emb, emb_dim=emb_dim) w2i = np.load(root_dir + '/data/vocab.npy').item() global i2w i2w = {v: k for k, v in w2i.items()} #amazon = Amazon_loader(dom=root_dir+domain, emb_dim=300) model = CNN(amazon.emb_dim, amazon.vocab_size, h_dim=args.h_dim, pretrained_emb=amazon.vectors, gpu=args.gpu) solver = optim.Adam(model.parameters(), lr=args.lr) if args.gpu: model.cuda() perf_dict[emb+':'+'amazonWE'] = run_model(amazon, model, solver) test_out = test(model, amazon) test_out = ['positive' if s > 0.5 else 'negative' for s in test_out] np.save(root_dir + 'test_out'+emb+'npy', test_out) for k, v in perf_dict.items(): print (k, v) print ("===============")
def main(): # Data Loader (Input Pipeline) print('loading dataset...') train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=False) # Define models print('building model...') cnn1 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn1.cuda() print(cnn1.parameters) optimizer1 = torch.optim.Adam(cnn1.parameters(), lr=learning_rate) cnn2 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn2.cuda() print(cnn2.parameters) optimizer2 = torch.optim.Adam(cnn2.parameters(), lr=learning_rate) cnn3 = CNN(input_channel=input_channel, n_outputs=num_classes) cnn3.cuda() print(cnn3.parameters) optimizer3 = torch.optim.Adam(cnn3.parameters(), lr=learning_rate) mean_pure_ratio1 = 0 mean_pure_ratio2 = 0 mean_pure_ratio3 = 0 with open(txtfile, "a") as myfile: myfile.write( 'epoch: train_acc1 train_acc2 train_acc3 test_acc1 test_acc2 test_acc3 pure_ratio1 pure_ratio2\n' ) epoch = 0 train_acc1 = 0 train_acc2 = 0 train_acc3 = 0 # evaluate models with random weights test_acc1, test_acc2, test_acc3 = evaluate(test_loader, cnn1, cnn2, cnn3) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Model3 %.4f %% Pure Ratio1 %.4f %% Pure Ratio2 %.4f %% Pure Ratio3 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, test_acc3, mean_pure_ratio1, mean_pure_ratio2, mean_pure_ratio3)) # save results with open(txtfile, "a") as myfile: output = '{:05d}: {:10.4f} {:10.4f} {:10.4f} {:9.4f} {:9.4f} {:9.4f} {:11.4f} {:11.4f} {:11.4f}\n'.format( epoch, train_acc1, train_acc2, train_acc3, test_acc1, test_acc2, test_acc3, mean_pure_ratio1, mean_pure_ratio2, mean_pure_ratio3) myfile.write(output) # myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n") best_acc = 0.0 # training for epoch in range(1, args.n_epoch): # train models cnn1.train() adjust_learning_rate(optimizer1, epoch) cnn2.train() adjust_learning_rate(optimizer2, epoch) cnn3.train() adjust_learning_rate(optimizer3, epoch) train_acc1, train_acc2, train_acc3, pure_ratio_1_list, pure_ratio_2_list, pure_ratio_3_list = train( train_loader, epoch, cnn1, optimizer1, cnn2, optimizer2, cnn3, optimizer3) # evaluate models test_acc1, test_acc2, test_acc3 = evaluate(test_loader, cnn1, cnn2, cnn3) if test_acc1 > best_acc: best_acc = test_acc1 torch.save(cnn1.state_dict(), '{}_best_epoch.pth'.format(args.dataset)) if test_acc2 > best_acc: best_acc = test_acc2 torch.save(cnn2.state_dict(), '{}_best_epoch.pth'.format(args.dataset)) if test_acc3 > best_acc: best_acc = test_acc3 torch.save(cnn3.state_dict(), '{}_best_epoch.pth'.format(args.dataset)) # save results mean_pure_ratio1 = sum(pure_ratio_1_list) / len(pure_ratio_1_list) mean_pure_ratio2 = sum(pure_ratio_2_list) / len(pure_ratio_2_list) mean_pure_ratio3 = sum(pure_ratio_3_list) / len(pure_ratio_3_list) print( 'Epoch [%d/%d] Test Accuracy on the %s test images: Model1 %.4f %% Model2 %.4f %% Model3 %.4f %%, Pure Ratio 1 %.4f %%, Pure Ratio 2 %.4f %% Pure Ratio 2 %.4f %%' % (epoch + 1, args.n_epoch, len(test_dataset), test_acc1, test_acc2, test_acc3, mean_pure_ratio1, mean_pure_ratio2, mean_pure_ratio3)) with open(txtfile, "a") as myfile: output = '{:05d}: {:10.4f} {:10.4f} {:10.4f} {:9.4f} {:9.4f} {:9.4f} {:11.4f} {:11.4f} {:11.4f}\n'.format( epoch, train_acc1, train_acc2, train_acc3, test_acc1, test_acc2, test_acc3, mean_pure_ratio1, mean_pure_ratio2, mean_pure_ratio2) myfile.write(output) # myfile.write(str(int(epoch)) + ': ' + str(train_acc1) +' ' + str(train_acc2) +' ' + str(test_acc1) + " " + str(test_acc2) + ' ' + str(mean_pure_ratio1) + ' ' + str(mean_pure_ratio2) + "\n") print('best acc: {}'.format(best_acc)) with open(txtfile, "a") as myfile: myfile.write('===>>> best acc: {} <<<==='.format(best_acc))
def main(args): print(sys.argv) if not os.path.exists('models'): os.mkdir('models') num_epochs = args.ne lr_decay = args.decay learning_rate = args.lr data_loader = get_data_loader(args.patches_path, args.gt_path, args.bs, num_workers=8) model = CNN() if torch.cuda.is_available(): model.cuda() model.train() if args.rms: optimizer = optim.RMSprop(model.parameters(), lr=args.lr, momentum=args.mm) else: optimizer = optim.Adam(model.parameters(), lr=args.lr) model_loss = torch.nn.CrossEntropyLoss() losses = [] try: for epoch in range(num_epochs): if epoch % args.decay_epoch == 0 and epoch > 0: learning_rate = learning_rate * lr_decay for param_group in optimizer.param_groups: param_group['lr'] = learning_rate loss_epoch = [] for step, (patches, gt) in enumerate(data_loader): if torch.cuda.is_available(): patches = patches.cuda() gt = gt.cuda() model.zero_grad() out = model(patches) loss = model_loss(out, gt) loss.backward() optimizer.step() loss_step = loss.cpu().detach().numpy() loss_epoch.append(loss_step) print('Epoch ' + str(epoch + 1) + '/' + str(num_epochs) + ' - Step ' + str(step + 1) + '/' + str(len(data_loader)) + " - Loss: " + str(loss_step)) loss_epoch_mean = np.mean(np.array(loss_epoch)) losses.append(loss_epoch_mean) print('Total epoch loss: ' + str(loss_epoch_mean)) if (epoch + 1) % args.save_epoch == 0 and epoch > 0: filename = 'model-epoch-' + str(epoch + 1) + '.pth' model_path = os.path.join('models/', filename) torch.save(model.state_dict(), model_path) except KeyboardInterrupt: pass filename = 'model-epoch-last.pth' model_path = os.path.join('models', filename) torch.save(model.state_dict(), model_path) plt.plot(losses) plt.show()
import numpy as np import torch import torch.nn as nn import torch.optim as optim from torch.autograd import Variable from model import CNN from data import KddData # 神经网络参数 batch_size = 128 learning_rate = 1e-2 num_epoches = 20 USE_GPU = torch.cuda.is_available() dataset = KddData(batch_size) model = CNN(1, 23) if USE_GPU: model = model.cuda() def train(): criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=learning_rate) for epoch in range(num_epoches): print('epoch {}'.format(epoch + 1)) print('*' * 10) running_loss = 0.0 running_acc = 0.0 for i, data in enumerate(dataset.train_dataloader, 1): img, label = data if USE_GPU:
class Classifier: def __init__(self, ds_path, lr, iterations, batch_size, hidden_layers_out, print_freq, save_dir, momentum, dropout): self.train_data = torchvision.datasets.MNIST(ds_path, train=True, transform=transforms.ToTensor(), download=True) self.test_data = torchvision.datasets.MNIST(ds_path, train=False, transform=transforms.ToTensor(), download=True) self.train_loader = torch.utils.data.DataLoader(self.train_data, batch_size=batch_size, shuffle=True) self.test_loader = torch.utils.data.DataLoader(self.test_data, batch_size=batch_size) self.save_dir = save_dir self.is_momentum = (momentum != 0.0) # Set Model Hyperparameters self.learning_rate = lr self.iterations = iterations self.print_freq = print_freq self.model = CNN(hidden_layers_out, dropout=dropout) self.cuda = torch.cuda.is_available() if self.cuda: self.model = self.model.cuda() def train(self, momentum, nesterov, weight_decay): train_loss_hist = [] train_acc_hist = [] test_loss_hist = [] test_acc_hist = [] best_pred = 0.0 end = time.time() for itr in range(self.iterations): self.model.train() if self.is_momentum: optimizer = optim.SGD(self.model.parameters(), lr=self.learning_rate, momentum=momentum, nesterov=nesterov, weight_decay=weight_decay) else: optimizer = optim.SGD(self.model.parameters(), lr=self.learning_rate) losses = AverageMeter() batch_time = AverageMeter() top1 = AverageMeter() for i, (x_batch, y_batch) in enumerate(self.train_loader): # Compute output for example logits = self.model(x_batch) loss = self.model.loss(logits, y_batch) # Update Mean loss for current iteration losses.update(loss.item(), x_batch.size(0)) prec1 = self.accuracy(logits.data, y_batch, k=1) top1.update(prec1.item(), x_batch.size(0)) # compute gradient and do SGD step loss.backward() optimizer.step() # Set grads to zero for new iter optimizer.zero_grad() batch_time.update(time.time() - end) end = time.time() if i % self.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {top1.val:.3f} ({top1.avg:.3f})'.format( itr, i, len(self.train_loader), batch_time=batch_time, loss=losses, top1=top1)) # evaluate on validation set test_loss, test_prec1 = self.test(self.test_loader) train_loss_hist.append(losses.avg) train_acc_hist.append(top1.avg) test_loss_hist.append(test_loss) test_acc_hist.append(test_prec1) # Store best model is_best = best_pred < test_prec1 if is_best: best_pred = test_prec1 self.save_checkpoint(is_best, (itr+1), self.model.state_dict(), self.save_dir) return (train_loss_hist, train_acc_hist, test_loss_hist, test_acc_hist) def test(self, batch_loader): self.model.eval() losses = AverageMeter() batch_time = AverageMeter() top1 = AverageMeter() end = time.time() for i, (x_batch,y_batch) in enumerate(batch_loader): with torch.no_grad(): logits = self.model(x_batch) loss = self.model.loss(logits, y_batch) # Update Mean loss for current iteration losses.update(loss.item(), x_batch.size(0)) prec1 = self.accuracy(logits.data, y_batch, k=1) top1.update(prec1.item(), x_batch.size(0)) batch_time.update(time.time() - end) end = time.time() if i % self.print_freq == 0: print('Epoch: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {top1.val:.3f} ({top1.avg:.3f})'.format( i, len(batch_loader), batch_time=batch_time, loss=losses, top1=top1)) print(' * Acc {top1.avg:.3f}'.format(top1=top1)) return (losses.avg, top1.avg) def accuracy(self, output, y, k=1): """Computes the precision@k for the specified values of k""" # Rehape to [N, 1] target = y.view(-1, 1) _, pred = torch.topk(output, k, dim=1, largest=True, sorted=True) correct = torch.eq(pred, target) return torch.sum(correct).float() / y.size(0) def save_checkpoint(self, is_best, epoch, state, save_dir, base_name="chkpt_plain"): """Saves checkpoint to disk""" directory = save_dir filename = base_name + ".pth.tar" if not os.path.exists(directory): os.makedirs(directory) filename = directory + filename torch.save(state, filename) if is_best: shutil.copyfile(filename, directory + base_name + '__model_best.pth.tar')
from board import Board # Load models # MODEL_CNN_PATH = os.path.join('models','model_n0.pt') MODEL_CNN_PATH = os.path.join('models','model_n1.pt') MODEL_QLEARN_PATH = os.path.join('models','model_q1.pkl') IG.create_alfabet() cnn = CNN() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") cnn.load_state_dict(torch.load(MODEL_CNN_PATH)) if device != "cpu": cnn.cuda(0) if os.path.exists(MODEL_QLEARN_PATH): with open(MODEL_QLEARN_PATH, 'rb') as f: qlearn = pickle.load(f) else: qlearn = {} # Game functions exitTerms = ("quit", "exit", "bye","q") def main(gameNumber=0, VERBOSE = True, SAVE_IMG = False, GAME_TYPE=None, QLEARN = False): PLAYER_TYPES = {