def train_iters(model, train_feature_path, val_feature_path, learning_rate=0.01, batch_size=20, epochs=50): criterion = nn.MultiLabelSoftMarginLoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5) # scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.95) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min') train_loss, val_loss, avg_loss = 0, 0, 0 train_accuracy, val_accuracy = 0, 0 num_iter = len(train_feature_path) // batch_size train_feature_vec, train_label_vec = load_features_labels( train_feature_path) val_feature_vec, val_label_vec = load_features_labels(val_feature_path) print('*** loaded train and val features') for i in range(epochs): for j in range(num_iter): x = Variable( torch.FloatTensor( np.array(train_feature_vec[j * batch_size:(j + 1) * batch_size]))).cuda() y = Variable( torch.FloatTensor( np.array(train_label_vec[j * batch_size:(j + 1) * batch_size]))).cuda() optimizer.zero_grad() y_pred = model(x) loss = criterion(y_pred, y) avg_loss += float(loss) loss.backward() optimizer.step() del x, y, y_pred if (j + 1) % 1000 == 0: train_loss = avg_loss / 1000 avg_loss = 0 val_loss = validate(model, val_feature_vec, val_label_vec) scheduler.step(val_loss) print('epoch %d: %d --- %f, %f' % (i, j, train_loss, val_loss)) writer.add_scalar('loss/train', train_loss, i) writer.add_scalar('loss/validate', val_loss, i) if (j + 1) % 5000 == 0: torch.save( model, 'output/trained_model_' + str(epochs) + '_' + str(learning_rate) + '_' + str(j)) train_accuracy = find_accuracy(model, train_feature_vec, train_label_vec) val_accuracy = find_accuracy(model, val_feature_vec, val_label_vec) print('----- validation accuracy: %f, train_accuracy: %f' % (val_accuracy, train_accuracy)) writer.add_scalar('accuracy/train', train_accuracy) writer.add_scalar('accuracy/val', val_accuracy) torch.save(model, 'output/final_model_' + str(epochs) + '_' + str(learning_rate)) return train_accuracy, val_accuracy
def train(): transforms = Compose([ToTensor()]) train_dataset = CaptchaData('./data/train/', transform=transforms) train_data_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=0, shuffle=True, drop_last=True) test_data = CaptchaData('./data/test/', transform=transforms) test_data_loader = DataLoader(test_data, batch_size=batch_size, num_workers=0, shuffle=True, drop_last=True) cnn = CNN() if torch.cuda.is_available(): cnn.cuda() if restor: cnn.load_state_dict(torch.load(model_path)) # freezing_layers = list(cnn.named_parameters())[:10] # for param in freezing_layers: # param[1].requires_grad = False # print('freezing layer:', param[0]) optimizer = torch.optim.Adam(cnn.parameters(), lr=base_lr) criterion = nn.MultiLabelSoftMarginLoss() for epoch in range(max_epoch): start_ = time.time() loss_history = [] acc_history = [] cnn.train() for img, target in train_data_loader: img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) loss = criterion(output, target) optimizer.zero_grad() loss.backward() optimizer.step() acc = calculat_acc(output, target) acc_history.append(float(acc)) loss_history.append(float(loss)) print('train_loss: {:.4}|train_acc: {:.4}'.format( torch.mean(torch.Tensor(loss_history)), torch.mean(torch.Tensor(acc_history)), )) loss_history = [] acc_history = [] cnn.eval() for img, target in test_data_loader: img = Variable(img) target = Variable(target) if torch.cuda.is_available(): img = img.cuda() target = target.cuda() output = cnn(img) acc = calculat_acc(output, target) acc_history.append(float(acc)) loss_history.append(float(loss)) print('test_loss: {:.4}|test_acc: {:.4}'.format( torch.mean(torch.Tensor(loss_history)), torch.mean(torch.Tensor(acc_history)), )) print('epoch: {}|time: {:.4f}'.format(epoch, time.time() - start_)) torch.save(cnn.state_dict(), model_path)
def train(fold: int, verbose: int = 30): # create folds df = split_dataset('data/dirty_mnist_2nd_answer.csv') df_train = df[df['kfold'] != fold].reset_index(drop=True) df_valid = df[df['kfold'] == fold].reset_index(drop=True) df_train.drop(['kfold'], axis=1).to_csv(f'data/train-kfold-{fold}.csv', index=False) df_valid.drop(['kfold'], axis=1).to_csv(f'data/valid-kfold-{fold}.csv', index=False) trainset = MnistDataset('data/dirty_mnist_2nd', f'data/train-kfold-{fold}.csv', transforms_train) train_loader = DataLoader(trainset, batch_size=64, shuffle=True, num_workers=0) validset = MnistDataset('data/dirty_mnist_2nd', f'data/valid-kfold-{fold}.csv', transforms_test) valid_loader = DataLoader(validset, batch_size=16, shuffle=False, num_workers=0) num_epochs = 10 device = 'cuda' scaler = GradScaler() model = EfficientNetModel().to(device) if fold == 0: model.load_state_dict(torch.load(f'models/effi_b4_kfold_e30/effinet_b4-f{fold}-48.pth')) else: model.load_state_dict(torch.load(f'models/effi_b4_kfold_e30/effinet_b4-f{fold}-28.pth')) base_optimizer = torch.optim.SGD optimizer = SAM(model.parameters(), base_optimizer, lr=0.001, momentum=0.9) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size = 5, gamma = 0.75) criterion = nn.MultiLabelSoftMarginLoss() for epoch in range(num_epochs): model.train() for i, (images, targets) in enumerate(train_loader): optimizer.zero_grad() images = images.to(device) targets = targets.to(device) with autocast(): outputs = model(images) loss = criterion(outputs, targets) loss.backward() # sam optimizer first_steop optimizer.first_step(zero_grad=True) # sam optimizer second_steop criterion(model(images), targets).backward() optimizer.second_step(zero_grad=True) if (i+1) % verbose == 0: outputs = outputs > 0.0 acc = (outputs == targets).float().mean() print(f'Fold {fold} | Epoch {epoch} | L: {loss.item():.7f} | A: {acc:.7f}') model.eval() valid_acc = 0.0 valid_loss = 0.0 for i, (images, targets) in enumerate(valid_loader): images = images.to(device) targets = targets.to(device) with autocast(): outputs = model(images) loss = criterion(outputs, targets) valid_loss += loss.item() outputs = outputs > 0.0 valid_acc += (outputs == targets).float().mean() print(f'Fold {fold} | Epoch {epoch} | L: {valid_loss/(i+1):.7f} | A: {valid_acc/(i+1):.7f}\n') # Learning rate 조절 lr_scheduler.step() if epoch > num_epochs-20 and epoch < num_epochs-1: torch.save(model.state_dict(), f'models/effinet_b4_SAM-f{fold}-{epoch}.pth')
resnet50(pretrained=True) ] i = 0 model = models[0] ## set up paths save_model_path = "{}/{}_{}_state_dict.pkl".format(root, root, model_names[i]) save_mat_path_fc = "{}/{}_{}_data_fc.mat".format(root, root, model_names[i]) save_mat_path_tune = "{}/{}_{}_data_tune.mat".format(root, root, model_names[i]) ## resize last fully connected layer to match our problem model.fc = nn.Linear(model.fc.in_features, 17) model.type(dtype) loss_fn = nn.MultiLabelSoftMarginLoss().type(dtype) state_dict = torch.load(save_model_path, map_location=lambda storage, loc: storage) model.load_state_dict(state_dict) print("{} model loaded from {}".format(model_names[i], os.path.abspath(save_model_path))) ## first optize sigmoid thresholds print("optimizing sigmoid cutoffs for each class") sig_scores, y_array = get_scores(model, train_loaders[i], dtype) sigmoid_threshold = optimize_F2(sig_scores, y_array) print("optimal thresholds: ", sigmoid_threshold)
def __init__(self, label_embed): super(SigmoidMLBCE, self).__init__() self.criterion = nn.MultiLabelSoftMarginLoss() self.num_class, self.embed_dim = label_embed.size() self.embeddings_t = torch.t(label_embed)
def multi_criterion(logits, labels): loss = nn.MultiLabelSoftMarginLoss()(logits, Variable(labels)) return loss
def main(): args = get_args() if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) # number of classes for each dataset. if args.dataset == 'PascalVOC': num_classes = 20 else: raise Exception("No dataset named {}.".format(args.dataset)) # Select Model & Method model = models.__dict__[args.arch](pretrained=args.pretrained, num_classes=num_classes) if args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) # define loss function (criterion) and optimizer criterion = nn.MultiLabelSoftMarginLoss().cuda(args.gpu) # criterion = nn.BCEWithLogitsLoss().cuda(args.gpu) # Take apart parameters to give different Learning Rate param_features = [] param_classifiers = [] if args.arch.startswith('vgg'): for name, parameter in model.named_parameters(): if 'features.' in name: param_features.append(parameter) else: param_classifiers.append(parameter) elif args.arch.startswith('resnet'): for name, parameter in model.named_parameters(): if 'layer4.' in name or 'fc.' in name: param_classifiers.append(parameter) else: param_features.append(parameter) else: raise Exception("Fail to recognize the architecture") # Optimizer optimizer = torch.optim.SGD([ {'params': param_features, 'lr': args.lr}, {'params': param_classifiers, 'lr': args.lr * args.lr_ratio}], momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nest) # optionally resume from a checkpoint if args.resume: model, optimizer = load_model(model, optimizer, args) train_loader, val_loader, test_loader = data_loader(args) saving_dir = os.path.join(args.log_folder, args.name) if args.evaluate: # test_ap, test_loss = evaluate_cam(val_loader, model, criterion, args) # test_ap, test_loss = evaluate_cam2(val_loader, model, criterion, args) test_ap, test_loss = evaluate_cam3(val_loader, model, criterion, args) print_progress(test_ap, test_loss, 0, 0, prefix='test') return # Training Phase best_m_ap = 0 for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch, args) # Train for one epoch train_ap, train_loss = \ train(train_loader, model, criterion, optimizer, epoch, args) print_progress(train_ap, train_loss, epoch+1, args.epochs) # Evaluate classification val_ap, val_loss = validate(val_loader, model, criterion, epoch, args) print_progress(val_ap, val_loss, epoch+1, args.epochs, prefix='validation') # # Save checkpoint at best performance: is_best = val_ap.mean() > best_m_ap if is_best: best_m_ap = val_ap.mean() save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_m_ap': best_m_ap, 'optimizer': optimizer.state_dict(), }, is_best, saving_dir) save_progress(saving_dir, train_ap, train_loss, val_ap, val_loss, args)
def loss(self, output, labels): criterion = nn.MultiLabelSoftMarginLoss() return criterion(output, labels)
def main(args): # To reproduce training results torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) # Create model directory if not os.path.exists(args.model_path): os.makedirs(args.model_path) # Image Preprocessing # For normalization, see https://github.com/pytorch/vision#models transform = transforms.Compose([ transforms.RandomCrop(args.crop_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) data_loader = get_loader(args.image_dir, args.file_list, transform, args.batch_size, True) model = EncoderCNN_singleimage_frontal(args.num_classes) if args.pretrained: model_pretrained = torch.load(args.pretrained) model.load_state_dict(model_pretrained, strict=False) # Get starting epoch #, note that model is named as '...your path to model/algoname-epoch#.pkl' # A little messy here. start_epoch = int( args.pretrained.split('/')[-1].split('-')[1].split('.')[0]) + 1 else: start_epoch = 1 # mult_label loss #bce_loss = nn.BCELoss() MIL_loss = nn.MultiLabelSoftMarginLoss() # changed to GPU mode if available if torch.cuda.is_available(): model.cuda() #bce_loss.cuda() MIL_loss.cuda() #dp_model = torch.nn.DataParallel(model, device_ids=[0,1]) # Train the model total_step = len(data_loader) print('len(data_loader)', total_step) learning_rate = args.learning_rate # starting training max_ap = 0 for epoch in range(start_epoch, args.num_epochs + 1): # start decay learning rate if epoch > args.lr_decay: frac = (epoch - args.lr_decay) / args.learning_rate_decay_every decay_rator = math.pow(0.8, frac) # decay the learning rate learning_rate = args.learning_rate * decay_rator print('Learning rate for Epoch %d: %.6f' % (epoch, learning_rate)) # Constructing model parameters for optimization optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(args.alpha, args.beta)) print('-------------Train for Epoch %d --------------' % epoch) for i, (images, image_name, image_labels) in enumerate(data_loader): images = to_var(images) image_labels = torch.FloatTensor(image_labels) image_labels = to_var(image_labels) model.train() #dp_model.train() optimizer.zero_grad() outputs = model(images) loss = MIL_loss(outputs, image_labels) loss.backward() optimizer.step() # Print log info if i % args.log_step == 0: print('Epoch [%d/%d], Step [%d/%d], BCE Loss: %.4f' % (epoch, args.num_epochs, i, total_step, loss.item())) gts = [] outputs = [] with torch.no_grad(): for i, (images, image_name, image_labels) in enumerate(data_loader): images = to_var(images) model.eval() #dp_model.train() output = model(images) output = sigmoid(output) output = output.data.cpu().numpy() gts.append(image_labels) outputs.append(output) if i == 96: break outputs = np.stack(outputs, 1).reshape(-1, 82) gts = np.stack(gts, 1).reshape(-1, 82) # outputs = outputs.data.cpu().numpy() # image_labels = image_labels.data.cpu().numpy() outs = [] for cls in range(82): #print(roc_auc_score( gts[:,cls],predictions[:,cls])) ap = voc_eval(cls, outputs, gts) outs.append(ap) print(np.mean(outs)) print(outs) # if np.mean(outs) > 0.13: # np.savetxt(args.predictions, outputs) # np.savetxt(args.gts, gts) # return # if epoch % 1 == 0: if np.mean(outs) > max_ap: max_ap = np.mean(outs) # Save the model after each epoch torch.save( model.state_dict(), os.path.join(args.model_path, 'Resnet_single_image-%d.pkl' % (epoch)))
def __init__(self): super(LogLoss, self).__init__() self.classify_loss = nn.MultiLabelSoftMarginLoss() self.__classes_num = NUM_CLASSES
def __init__(self, H_PARAM, embed_layer, bert=None): super(KeyWordPredictor, self).__init__() self.N_word = H_PARAM['N_WORD'] self.N_depth = H_PARAM['N_depth'] self.N_h = H_PARAM['N_h'] self.gpu = H_PARAM['gpu'] self.use_hs = H_PARAM['use_hs'] self.table_type = H_PARAM['table_type'] self.acc_num = 3 self.embed_layer = embed_layer self.use_bert = True if bert else False if bert: self.q_bert = bert encoded_num = 768 else: self.q_lstm = nn.LSTM(input_size=self.N_word, hidden_size=self.N_h // 2, num_layers=self.N_depth, batch_first=True, dropout=0.3, bidirectional=True) encoded_num = self.N_h self.hs_lstm = nn.LSTM(input_size=self.N_word, hidden_size=self.N_h // 2, num_layers=self.N_depth, batch_first=True, dropout=0.3, bidirectional=True) self.kw_lstm = nn.LSTM(input_size=self.N_word, hidden_size=self.N_h // 2, num_layers=self.N_depth, batch_first=True, dropout=0.3, bidirectional=True) self.q_num_att = nn.Linear(encoded_num, self.N_h) self.hs_num_att = nn.Linear(self.N_h, self.N_h) self.kw_num_out_q = nn.Linear(encoded_num, self.N_h) self.kw_num_out_hs = nn.Linear(self.N_h, self.N_h) self.kw_num_out = nn.Sequential(nn.Tanh(), nn.Linear(self.N_h, 4)) # num of key words: 0-3 self.q_att = nn.Linear(encoded_num, self.N_h) self.hs_att = nn.Linear(self.N_h, self.N_h) self.kw_out_q = nn.Linear(encoded_num, self.N_h) self.kw_out_hs = nn.Linear(self.N_h, self.N_h) self.kw_out_kw = nn.Linear(self.N_h, self.N_h) self.kw_out = nn.Sequential(nn.Tanh(), nn.Linear(self.N_h, 1)) self.softmax = nn.Softmax() #dim=1 self.CE = nn.CrossEntropyLoss() self.log_softmax = nn.LogSoftmax() self.mlsml = nn.MultiLabelSoftMarginLoss() self.bce_logit = nn.BCEWithLogitsLoss() self.sigm = nn.Sigmoid() if self.gpu: self.cuda()
def __init__(self, H_PARAM, embed_layer): super(SuperModel, self).__init__() self.train_component = H_PARAM['train_component'] self.gpu = H_PARAM['gpu'] self.B_word = H_PARAM['B_WORD'] self.N_word = H_PARAM['N_WORD'] self.N_h = H_PARAM['N_h'] self.N_depth = H_PARAM['N_depth'] self.trainable_emb = H_PARAM['trainable_emb'] self.table_type = H_PARAM['table_type'] self.use_hs = H_PARAM['use_hs'] self.with_from = H_PARAM['with_from'] self.SQL_TOK = [ '<UNK>', '<END>', 'WHERE', 'AND', 'EQL', 'GT', 'LT', '<BEG>' ] self.use_bert = H_PARAM['use_bert'] #False self.table_type = H_PARAM['table_type'] self.acc_num = 0 self.table_dict = None self.test_db_ids = set() self.save_dir = H_PARAM['save_dir'] # word embedding layer self.embed_layer = embed_layer # initial all modules self.multi_sql = MultiSqlPredictor(H_PARAM) self.multi_sql.eval() self.key_word = KeyWordPredictor(H_PARAM) self.key_word.eval() self.col = ColPredictor(H_PARAM) self.col.eval() self.op = OpPredictor(H_PARAM) self.op.eval() self.agg = AggPredictor(H_PARAM) self.agg.eval() self.root_teminal = RootTeminalPredictor(H_PARAM) self.root_teminal.eval() self.des_asc = DesAscLimitPredictor(H_PARAM) self.des_asc.eval() self.having = HavingPredictor(H_PARAM) self.having.eval() self.andor = AndOrPredictor(H_PARAM) self.andor.eval() self.from_table = FromPredictor(H_PARAM) self.from_table.eval() self.softmax = nn.Softmax() #dim=1 self.CE = nn.CrossEntropyLoss() self.log_softmax = nn.LogSoftmax() self.mlsml = nn.MultiLabelSoftMarginLoss() self.bce_logit = nn.BCEWithLogitsLoss() self.sigm = nn.Sigmoid() if self.gpu: self.cuda() self.path_not_found = 0
def main(): global args, best_prec1 args = parser.parse_args() args.distributed = args.world_size > 1 # create model print("=> creating model '{}'".format(args.arch)) if args.arch == 'localizer_alexnet': model = localizer_alexnet(pretrained=args.pretrained) elif args.arch == 'localizer_alexnet_robust': model = localizer_alexnet_robust(pretrained=args.pretrained) print(model) model.features = torch.nn.DataParallel(model.features) model.cuda() # TODO: # define loss function (criterion) and optimizer criterion = nn.MultiLabelSoftMarginLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) #optimizer = torch.optim.Adam(model.parameters(), lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True random.seed( 100 ) #Fixing seed so that dataloader generates the same random batches always. np.random.seed(100) # Data loading code # TODO: Write code for IMDBDataset in custom.py trainval_imdb = get_imdb('voc_2007_trainval') test_imdb = get_imdb('voc_2007_test') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = IMDBDataset( trainval_imdb, transforms.Compose([ transforms.Resize((512, 512)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(IMDBDataset( test_imdb, transforms.Compose([ transforms.Resize((384, 384)), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return # TODO: Create loggers for visdom and tboard # TODO: You can pass the logger objects to train(), make appropriate # modifications to train() if args.vis: import visdom vis = visdom.Visdom( server='http://ec2-3-21-126-90.us-east-2.compute.amazonaws.com/', port='8097' ) #Change Address as per need. Can change to commandline arg else: vis = None timestr = time.strftime("%Y%m%d-%H%M%S") save_path = "runs/" + timestr + '/' writer = SummaryWriter(save_path) #Initializing model weights pretrained_alexnet = models.alexnet(pretrained=True) conv_layer_numbers = [0, 3, 6, 8, 10] for elt in conv_layer_numbers: #print(model.features.module[elt]) model.features.module[elt].weight.data.copy_( pretrained_alexnet.features[elt].weight.data) model.features.module[elt].bias.data.copy_( pretrained_alexnet.features[elt].bias.data) xavier_initialized_conv_layers = [0, 2, 4] for elt in xavier_initialized_conv_layers: torch.nn.init.xavier_uniform_(model.classifier[elt].weight) print("Registering backward hook for final conv layer") model.classifier[4].register_backward_hook(hook) iter_cnt = 0 for epoch in range(args.start_epoch, args.epochs): print("Epoch count is: ", epoch, "\t", "Iteration Count is: ", iter_cnt) adjust_learning_rate(optimizer, epoch) # train for one epoch iter_cnt = train(train_loader, model, criterion, optimizer, epoch, iter_cnt, writer, vis) #Obtaining histogram of weights and histogram of gradients of weights '''for i,elt in enumerate(conv_layer_numbers): weight_tag = "feature_conv_" + str(i) + "_weight" grad_tag = "feature_conv_" + str(i) + "_grad" pdb.set_trace() writer.add_histogram(weight_tag, model.features.module[elt].weight.data.cpu(), epoch) writer.add_histogram(grad_tag, model.features.module[elt].weight.grad.data.cpu(), epoch) for i,elt in enumerate(xavier_initialized_conv_layers): weight_tag = "classifier_conv_" + str(i+len(conv_layer_numbers)) + "_weight" grad_tag = "classifier_conv_" + str(i+len(conv_layer_numbers)) + "_grad" writer.add_histogram(weight_tag, model.classifier[elt].weight.data.cpu(), epoch) writer.add_histogram(grad_tag, model.classifier[elt].weight.grad.data.cpu(), epoch)''' # evaluate on validation set if epoch % args.eval_freq == 0 or epoch == args.epochs - 1: print("Evaluating on Validation Set. Epoch number is: ", epoch) m1, m2 = validate(val_loader, model, criterion, epoch, writer) # score = m1 #TODO: Delete this line and uncomment the line below later score = m1 * m2 # remember best prec@1 and save checkpoint is_best = score > best_prec1 best_prec1 = max(score, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best) writer.close()
model = Net() model.cuda() params = list(model.parameters()) print(model) print('[init] number of parameters %d' % (len(params))) for i in params: print(i.size()) optimizer = optim.SGD(model.parameters(), lr=learn_rate_init, momentum=momentum_coeff) spweight = loadmat_transpose(path_weight) spweight = torch.from_numpy(spweight['ratio'][0] * 100).cuda() criterion = nn.MultiLabelSoftMarginLoss(weight=spweight).cuda() def dataset_dimension(index, select): #note: select can be in any order! [7, 3, 9, 11, 4, ...] var_samples = 0 var_width = index[1][1] lab_samples = 0 lab_width = index[1][3] for i in select: var_samples += index[i][0] lab_samples += index[i][2] return var_samples, var_width, lab_samples, lab_width
def main(argv): parser = argparse.ArgumentParser(description='CNN baseline for DSTC5 SAP Task') parser.add_argument('--trainset', dest='trainset', action='store', metavar='TRAINSET', required=True, help='') parser.add_argument('--testset', dest='testset', action='store', metavar='TESTSET', required=True, help='') parser.add_argument('--dataroot', dest='dataroot', action='store', required=True, metavar='PATH', help='') parser.add_argument('--roletype', dest='roletype', action='store', choices=['guide', 'tourist'], required=True, help='speaker') args = parser.parse_args() train_utters = [] trainset = dataset_walker.dataset_walker(args.trainset, dataroot=args.dataroot, labels=True, translations=True) sys.stderr.write('Loading training instances ... ') for call in trainset: for (log_utter, translations, label_utter) in call: if log_utter['speaker'].lower() != args.roletype: continue transcript = data_helpers.tokenize_and_lower(log_utter['transcript']) speech_act = label_utter['speech_act'] sa_label_list = [] for sa in speech_act: sa_label_list += ['%s_%s' % (sa['act'], attr) for attr in sa['attributes']] sa_label_list = sorted(set(sa_label_list)) train_utters += [(transcript, log_utter['speaker'], sa_label_list)] sys.stderr.write('Done\n') test_utters = [] testset = dataset_walker.dataset_walker(args.testset, dataroot=args.dataroot, labels=True, translations=True) sys.stderr.write('Loading testing instances ... ') for call in testset: for (log_utter, translations, label_utter) in call: if log_utter['speaker'].lower() != args.roletype: continue try: translation = data_helpers.tokenize_and_lower(translations['translated'][0]['hyp']) except: translation = '' speech_act = label_utter['speech_act'] sa_label_list = [] for sa in speech_act: sa_label_list += ['%s_%s' % (sa['act'], attr) for attr in sa['attributes']] sa_label_list = sorted(set(sa_label_list)) test_utters += [(translation, log_utter['speaker'], sa_label_list)] pprint(train_utters[:2]) pprint(test_utters[:2]) # load parameters params = data_helpers.load_params("parameters/cnn.txt") pprint(params) num_epochs = int(params['num_epochs']) validation_split = float(params['validation_split']) batch_size = int(params['batch_size']) multilabel = params['multilabel']=="true" # build vocabulary sents = [utter[0].split(' ') for utter in train_utters] max_sent_len = int(params['max_sent_len']) pad_sents = data_helpers.pad_sentences(sents, max_sent_len) vocabulary, inv_vocabulary = data_helpers.build_vocab(pad_sents) print("vocabulary size: %d" % len(vocabulary)) # params['max_sent_len'] = max_sent_len # build inputs train_inputs = data_helpers.build_input_data(pad_sents, vocabulary) test_sents = [utter[0].split(' ') for utter in test_utters] test_pad_sents = data_helpers.pad_sentences(test_sents, max_sent_len) test_inputs = data_helpers.build_input_data(test_pad_sents, vocabulary) # build labels sa_train_labels = [utter[2] for utter in train_utters] sa_test_labels = [utter[2] for utter in test_utters] label_binarizer = preprocessing.MultiLabelBinarizer() label_binarizer.fit(sa_train_labels+sa_test_labels) train_labels = label_binarizer.transform(sa_train_labels) test_labels = label_binarizer.transform(sa_test_labels) # split and shuffle data indices = np.arange(train_inputs.shape[0]) np.random.shuffle(indices) train_inputs = train_inputs[indices] train_labels = train_labels[indices] num_validation = int(validation_split * train_inputs.shape[0]) # x_train = train_inputs[:-num_validation] # y_train = train_labels[:-num_validation] # x_val = train_inputs[-num_validation:] # y_val = train_labels[-num_validation:] x_train = train_inputs y_train = train_labels x_test = test_inputs y_test_numpy = test_labels # construct a pytorch data_loader x_train = torch.from_numpy(x_train).long() y_train = torch.from_numpy(y_train).float() dataset_tensor = data_utils.TensorDataset(x_train, y_train) train_loader = data_utils.DataLoader(dataset_tensor, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=False) x_test = torch.from_numpy(x_test).long() y_test = torch.from_numpy(y_test_numpy).long() dataset_tensor = data_utils.TensorDataset(x_test, y_test) test_loader = data_utils.DataLoader(dataset_tensor, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=False) # load pre-trained word embeddings embedding_dim = int(params['embedding_dim']) embedding_matrix = data_helpers.load_embedding(vocabulary, embedding_dim=embedding_dim, embedding=params['embedding']) # load model model = SluConvNet(params, embedding_matrix, len(vocabulary), y_train.shape[1]) if torch.cuda.is_available(): model = model.cuda() learning_rate = float(params['learning_rate']) optimizer = optim.Adam(model.parameters(), lr=learning_rate) # loss_fn = nn.BCEWithLogitsLoss() loss_fn = nn.MultiLabelSoftMarginLoss() for epoch in range(num_epochs): model.train() # set the model to training mode (apply dropout etc) for i, (inputs, labels) in enumerate(train_loader): inputs, labels = autograd.Variable(inputs), autograd.Variable(labels) if torch.cuda.is_available(): inputs, labels = inputs.cuda(), labels.cuda() preds = model(inputs) if torch.cuda.is_available(): preds = preds.cuda() loss = loss_fn(preds, labels) optimizer.zero_grad() loss.backward() optimizer.step() if i % 100 == 0: print("current loss: %.4f" % loss) model.eval() # set the model to evaluation mode true_acts, pred_acts, metrics = evaluate(model, label_binarizer, test_loader, y_test_numpy) print("Precision: %.4f\tRecall: %.4f\tF1-score: %.4f\n" % (metrics[0], metrics[1], metrics[2])) # end of training true_acts, pred_acts, metrics = evaluate(model, label_binarizer, test_loader, y_test_numpy) print("Precision: %.4f\tRecall: %.4f\tF1-score: %.4f\n" % (metrics[0], metrics[1], metrics[2])) with open(("pred_result_%s.txt" % args.roletype), "w") as f: for pred_act, true_act in zip(pred_acts, true_acts): f.write("pred: %s\ntrue: %s\n\n" % (', '.join(pred_act), ', '.join(true_act)))
['glu', nn.GLU()], ]) loss = nn.ModuleDict([ ['l1', nn.L1Loss()], ['nll', nn.NLLLoss()], ['kldiv', nn.KLDivLoss()], ['mse', nn.MSELoss()], ['bce', nn.BCELoss()], ['bce_with_logits', nn.BCEWithLogitsLoss()], ['cosine_embedding', nn.CosineEmbeddingLoss()], ['ctc', nn.CTCLoss()], ['hinge_embedding', nn.HingeEmbeddingLoss()], ['margin_ranking', nn.MarginRankingLoss()], ['multi_label_margin', nn.MultiLabelMarginLoss()], ['multi_label_soft_margin', nn.MultiLabelSoftMarginLoss()], ['multi_margin', nn.MultiMarginLoss()], ['smooth_l1', nn.SmoothL1Loss()], ['soft_margin', nn.SoftMarginLoss()], ['cross_entropy', nn.CrossEntropyLoss()], ['triplet_margin', nn.TripletMarginLoss()], ['poisson_nll', nn.PoissonNLLLoss()] ]) def _parse( identifier: typing.Union[str, typing.Type[nn.Module], nn.Module], dictionary: nn.ModuleDict, target: str ) -> nn.Module: """
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.enabled = False model = MnistModel().to(device) print(summary(model, input_size=(1, 3, 256, 256), verbose=0)) # 학습하기 # 무진님이 주신 에러 대응 코드 사용! # 실행 하는 곳이 메인인 경우 if __name__ == '__main__': # 옵티마이저와 멀티라벨소프트 마진 로스를 사용함 optimizer = optim.Adam(model.parameters(), lr=1e-3) criterion = nn.MultiLabelSoftMarginLoss() # 에포치 10주고 모델을 트레인으로 변환 num_epochs = 40 model.train() # 에포치 만큼 반복 for epoch in range(num_epochs): # 배치 사이즈 만큼 스탭을 진행함 for i, (images, targets) in enumerate(train_loader): # 미분 값 초기화 optimizer.zero_grad() # 데이터셋을 프로세스에 입력함 images = images.to(device) targets = targets.to(device)
def main(opt): # Set the random seed manually for reproducibility. if torch.cuda.is_available(): torch.cuda.manual_seed(opt.seed) else: torch.manual_seed(opt.seed) train_opt = { 'label_file': opt.train_label, 'imageinfo_file': opt.train_imageinfo, 'image_dir': opt.train_image_dir, 'batch_size': opt.batch_size, 'num_workers': opt.num_workers, 'train': True } val_opt = { 'label_file': opt.val_label, 'imageinfo_file': opt.val_imageinfo, 'image_dir': opt.val_image_dir, 'batch_size': opt.batch_size, 'num_workers': opt.num_workers, 'train': False } train_loader = get_data_loader(train_opt) val_loader = get_data_loader(val_opt) logger.info('Building model...') num_labels = train_loader.dataset.get_num_labels() model = DepNet(num_labels, finetune=opt.finetune, cnn_type=opt.cnn_type) criterion = nn.MultiLabelSoftMarginLoss() optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate) if torch.cuda.is_available(): model.cuda() criterion.cuda() logger.info("Start training...") best_loss = sys.maxsize best_epoch = 0 for epoch in range(opt.num_epochs): learning_rate = adjust_learning_rate(opt, optimizer, epoch) logger.info('===> Learning rate: %f: ', learning_rate) # train for one epoch train(opt, model, criterion, optimizer, train_loader, epoch) # validate at every val_step epoch if epoch % opt.val_step == 0: logger.info("Start evaluating...") val_loss, val_score = test(opt, model, criterion, val_loader) logger.info('Val loss: \n%s', val_loss) logger.info('Val score: \n%s', val_score) loss = val_loss.avg if loss < best_loss: logger.info('Found new best loss: %.7f, previous loss: %.7f', loss, best_loss) best_loss = loss best_epoch = epoch logger.info('Saving new checkpoint to: %s', opt.output_file) torch.save( { 'epoch': epoch, 'model': model.state_dict(), 'best_loss': best_loss, 'best_epoch': best_epoch, 'opt': opt }, opt.output_file) else: logger.info('Current loss: %.7f, best loss is %.7f @ epoch %d', loss, best_loss, best_epoch) if epoch - best_epoch > opt.max_patience: logger.info('Terminated by early stopping!') break
def __init__(self, N_word, N_h, N_depth, gpu, use_hs, dr=0.3, temperature=False): super(KeyWordPredictor, self).__init__() self.N_h = N_h self.gpu = gpu self.use_hs = use_hs self.q_lstm = nn.LSTM(input_size=N_word, hidden_size=N_h / 2, num_layers=N_depth, batch_first=True, dropout=dr, bidirectional=True) self.hs_lstm = nn.LSTM(input_size=N_word, hidden_size=N_h / 2, num_layers=N_depth, batch_first=True, dropout=dr, bidirectional=True) self.kw_lstm = nn.LSTM(input_size=N_word, hidden_size=N_h / 2, num_layers=N_depth, batch_first=True, dropout=dr, bidirectional=True) self.q_num_att = nn.Linear(N_h, N_h) self.hs_num_att = nn.Linear(N_h, N_h) self.kw_num_out_q = nn.Linear(N_h, N_h) self.kw_num_out_hs = nn.Linear(N_h, N_h) self.kw_num_out = nn.Sequential(nn.Tanh(), nn.Linear(N_h, 4)) # num of key words: 0-3 self.q_att = nn.Linear(N_h, N_h) self.hs_att = nn.Linear(N_h, N_h) self.kw_out_q = nn.Linear(N_h, N_h) self.kw_out_hs = nn.Linear(N_h, N_h) self.kw_out_kw = nn.Linear(N_h, N_h) self.kw_out = nn.Sequential(nn.Tanh(), nn.Linear(N_h, 1)) self.softmax = nn.Softmax() #dim=1 self.CE = nn.CrossEntropyLoss() self.log_softmax = nn.LogSoftmax() self.mlsml = nn.MultiLabelSoftMarginLoss() self.bce_logit = nn.BCEWithLogitsLoss() self.sigm = nn.Sigmoid() if temperature: self.T1 = nn.Parameter(torch.FloatTensor([1.])) # kw_num_out self.T2 = nn.Parameter(torch.FloatTensor([1.])) # kw_out else: self.T1 = self.T2 = 1.0 if gpu: self.cuda()
def main(): cnn = CNN() #resume resume = './model_cos_90_0.01.pkl' checkpoint = torch.load(resume) cnn.load_state_dict(checkpoint) if use_cuda: cnn = CNN().cuda() cnn.train() print('init net') criterion = nn.MultiLabelSoftMarginLoss() optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) best_acc = 0 # Train the Model train_dataloader = my_dataset.get_finetune_data_loader() for epoch in range(num_epochs): correct = 0 total = 0 for i, (images, labels) in enumerate(train_dataloader): #lr_schedule lr = cosine_anneal_schedule(epoch) for param_group in optimizer.param_groups: #print(param_group['lr']) param_group['lr'] = lr images = Variable(images).cuda() labels = Variable(labels.float()).cuda() predict_labels = cnn(images) # print(predict_labels.type) # print(labels.type) loss = criterion(predict_labels, labels) optimizer.zero_grad() loss.backward() optimizer.step() predict_labels = predict_labels.cpu() labels = labels.cpu() c0 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, 0:captcha_setting.ALL_CHAR_SET_LEN].data.numpy())] c1 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, captcha_setting.ALL_CHAR_SET_LEN:2 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())] c2 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, 2 * captcha_setting.ALL_CHAR_SET_LEN:3 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())] c3 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, 3 * captcha_setting.ALL_CHAR_SET_LEN:4 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())] # c = '%s%s%s%s' % (c0, c1, c2, c3) # print(c) predict_labels = '%s%s%s%s' % (c0, c1, c2, c3) # predict = '%s%s%s%s' % (c0, c1, c2, c3) # predict_label = one_hot_encoding.decode(predict.numpy()[0]) true_label = one_hot_encoding.decode(labels.numpy()[0]) total += labels.size(0) print(predict_labels, true_label) if (predict_labels == true_label): correct += 1 acc = 100 * correct / total # if (total % 200 == 0): # print('Test Accuracy of the model on the %d train images: %f %%' % (total, 100 * correct / total)) if (i+1) % 10 == 0: print("epoch:", epoch, "step:", i, "loss:", loss.item(),"Accuracy:", acc) if acc >= best_acc: torch.save(cnn.state_dict(), "./model_cos_90_0.01_finetune_2.pkl") #current is model.pkl print("save model") #print("epoch:", epoch, "loss:", loss.item(),"Accuracy:", acc) #torch.save(cnn.state_dict(), "./model_cos_90_0.01_finetune.pkl") #current is model.pkl print("save last model")
nn.BatchNorm2d(512), nn.ReLU(inplace=True), nn.Conv2d(512, 1024, stride=1, kernel_size=3, padding=1), # 1024,22,8 nn.BatchNorm2d(1024), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=2, stride=2), # 1024,11,4 ) self.fc = nn.Sequential(nn.Linear(45056, 2048), nn.ReLU(inplace=True), nn.Linear(2048, 1024), nn.ReLU(inplace=True), nn.Linear(1024, 248)) def forward(self, x): x = self.layer1(x) x = self.layer2(x) x = self.layer3(x) x = self.layer4(x) x = x.view(x.size(0), -1) x = self.fc(x) return x net = CNN_Network() net.to(DEVICE) optimizer = torch.optim.Adam(net.parameters(), lr=lr) loss = nn.MultiLabelSoftMarginLoss() dummy_input = torch.rand(16, 1, 35, 90) model = CNN_Network() with SummaryWriter(comment='Net', log_dir='scalar') as w: w.add_graph(model, (dummy_input, ))
def _compute_bce_discriminator_loss(netD, real_imgs, fake_imgs, real_labels, fake_labels, real_catelabels, conditions, gpus): cate_criterion =nn.MultiLabelSoftMarginLoss() criterion = nn.BCELoss() ratio = 1.0 # gp_lambda = 0.00005 batch_size = real_imgs.size(0) cond = conditions.detach() fake = fake_imgs.detach() real_features = nn.parallel.data_parallel(netD, (real_imgs), gpus) fake_features = nn.parallel.data_parallel(netD, (fake), gpus) # real pairs inputs = (real_features, cond) real_logits = nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus) errD_real = criterion(real_logits, real_labels) # wrong pairs inputs = (real_features[:(batch_size-1)], cond[1:]) wrong_logits = \ nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus) errD_wrong = criterion(wrong_logits, fake_labels[1:]) # fake pairs inputs = (fake_features, cond) fake_logits = nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus) errD_fake = criterion(fake_logits, fake_labels) # gradient penalty # epsilon = torch.rand([len(real_imgs)] + [1 for _ in range(len(real_imgs.shape)-1)], \ # device='cuda', requires_grad=True) # grad = _get_gradient(netD, real_imgs, fake_imgs, epsilon).contiguous() # grad -= grad.min(1, keepdim=True)[0] # grad /= grad.max(1, keepdim=True)[0] # gp = _gradient_penalty(grad) if netD.get_uncond_logits is not None: real_logits = \ nn.parallel.data_parallel(netD.get_uncond_logits, (real_features), gpus) fake_logits = \ nn.parallel.data_parallel(netD.get_uncond_logits, (fake_features), gpus) uncond_errD_real = criterion(real_logits, real_labels) uncond_errD_fake = criterion(fake_logits, fake_labels) errD = ((errD_real + uncond_errD_real) / 2. + (errD_fake + errD_wrong + uncond_errD_fake) / 3.) errD_real = (errD_real + uncond_errD_real) / 2. errD_fake = (errD_fake + uncond_errD_fake) / 2. else: errD = errD_real + (errD_fake + errD_wrong) * 0.5 acc = 0 if netD.cate_classify is not None: cate_logits = nn.parallel.data_parallel(netD.cate_classify, real_features, gpus) cate_logits = cate_logits.squeeze() errD = errD + ratio * cate_criterion(cate_logits, real_catelabels) acc = accuracy_score(real_catelabels.cpu().data.numpy().astype('int32'), (cate_logits.cpu().data.numpy() > 0.5).astype('int32')) return errD, errD_real.data, errD_wrong.data, errD_fake.data, acc
if the_number_of_gpu > 1: log_func('[i] the number of gpu : {}'.format(the_number_of_gpu)) model = nn.DataParallel(model) # for sync bn # patch_replication_callback(model) load_model_fn = lambda: load_model( model, model_path, parallel=the_number_of_gpu > 1) save_model_fn = lambda: save_model( model, model_path, parallel=the_number_of_gpu > 1) ################################################################################### # Loss, Optimizer ################################################################################### class_loss_fn = nn.MultiLabelSoftMarginLoss(reduction='none').cuda() if args.re_loss == 'L1_Loss': re_loss_fn = L1_Loss else: re_loss_fn = L2_Loss log_func('[i] The number of pretrained weights : {}'.format( len(param_groups[0]))) log_func('[i] The number of pretrained bias : {}'.format( len(param_groups[1]))) log_func('[i] The number of scratched weights : {}'.format( len(param_groups[2]))) log_func('[i] The number of scratched bias : {}'.format( len(param_groups[3])))
######################################################################################## if __name__ == '__main__': print('%s: calling main function ... ' % os.path.basename(__file__)) # https://discuss.pytorch.org/t/print-autograd-graph/692/8 batch_size = 64 num_classes = 17 C, H, W = 3, 256, 256 inputs = torch.randn(batch_size, C, H, W) labels = torch.randn(batch_size, num_classes) in_shape = inputs.size()[1:] if 1: net = densenet121(in_shape=in_shape, num_classes=num_classes).cuda().train() x = Variable(inputs) logits, probs = net.forward(x.cuda()) loss = nn.MultiLabelSoftMarginLoss()(logits, Variable(labels.cuda())) loss.backward() print(type(net)) print(net) print('probs') print(probs) input('Press ENTER to continue.')
else: print("=> no checkpoint found at '{}'".format(args.resume)) if torch.cuda.device_count() > 1: print "Using %d gpus." % torch.cuda.device_count() generator = nn.DataParallel(generator) discriminator = nn.DataParallel(discriminator) if torch.cuda.is_available(): generator.cuda() discriminator.cuda() if args.verbosity >= 1: print 'Done!' if args.dual_discrim: class_loss_fn = nn.MultiLabelSoftMarginLoss() loss_fn = nn.BCELoss() D_total_fake_loss = 0 D_total_fake_binary_loss = 0 D_total_fake_class_loss = 0 D_total_real_loss = 0 D_total_real_binary_loss = 0 D_total_real_class_loss = 0 G_total_loss = 0 G_total_binary_loss = 0 G_total_class_loss = 0 for epoch in xrange(0):
def main(): global args, best_prec1 torch.manual_seed(157) torch.cuda.manual_seed(157) #np.random.seed(0) args = parser.parse_args() args.distributed = args.world_size > 1 # create model print("=> creating model '{}'".format(args.arch)) if args.arch == 'localizer_alexnet': model = localizer_alexnet(pretrained=args.pretrained) elif args.arch == 'localizer_alexnet_robust': model = localizer_alexnet_robust(pretrained=args.pretrained) args.lr = args.lr print(model) # torch.cuda.set_device(0) # print(torch.cuda.current_device()) model.features = torch.nn.DataParallel(model.features) model.cuda() # TODO: # define loss function (criterion) and optimizer criterion = nn.MultiLabelSoftMarginLoss().cuda() # criterion = nn.BCEWithLogitsLoss().cuda() # optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) optimizer = torch.optim.SGD(model.classifier.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code # TODO: Write code for IMDBDataset in custom.py # trainval_imdb.num_images = 5011 trainval_imdb = get_imdb('voc_2007_trainval') test_imdb = get_imdb('voc_2007_test') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = IMDBDataset( trainval_imdb, transforms.Compose([ transforms.Resize((512, 512)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_sampler = None # len(train_loader.dataset.imgs) = 5011 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=train_sampler) # len(val_loader.dataset.imgs) = 4952, dir(val_loader) = 'class_to_idx', 'classes', 'imdb', 'imgs', 'loader', 'target_transform', 'transform' val_loader = torch.utils.data.DataLoader(IMDBDataset( test_imdb, transforms.Compose([ transforms.Resize((384, 384)), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) # TODO: Create loggers for visdom and tboard # TODO: You can pass the logger objects to train(), make appropriate # modifications to train() if args.arch == 'localizer_alexnet': data_log = logger.Logger('./logs/task1', name='freeloc') vis = visdom.Visdom(server='http://localhost', port='8087') else: data_log = logger.Logger('./logs_robust/task1', name='freeloc') vis = visdom.Visdom(server='http://localhost', port='8087') if args.evaluate: validate(val_loader, model, criterion, vis, 0) return if args.arch == 'localizer_alexnet': args.epochs = 30 else: args.epochs = 45 for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch, data_log, vis, args.arch) # evaluate on validation set if epoch % 10 == 0 or epoch == args.epochs - 1: # if True: #temp print("now evaluating!!!!!!") m1, m2 = validate(val_loader, model, criterion, vis, epoch) score = m1 * m2 # remember best prec@1 and save checkpoint is_best = score > best_prec1 best_prec1 = max(score, best_prec1) data_log.scalar_summary(tag='valid/metric1', value=m1, step=epoch) data_log.scalar_summary(tag='valid/metric2', value=m2, step=epoch) # import pdb;pdb.set_trace() save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best) print('end of training')
def train(): imgModel = ImgModel() loss_fn = nn.MultiLabelSoftMarginLoss( ) # MultiLabelSoftMarginLoss # CrossEntropyLoss if torch.cuda.is_available(): # GPU imgModel.to(DEVICE) loss_fn.to(DEVICE) loss_fn = nn.MultiLabelSoftMarginLoss( ) # MultiLabelSoftMarginLoss # CrossEntropyLoss optimizer = optim.Adam(params=imgModel.parameters(), ) acces = [] epoch_flag = False train_img_dir = os.path.join(imgModel.img_dir, 'trainImg_l{}'.format(imgModel.char_len)) train_dataloder = data.DataLoader(dataset=ImgDataset( img_dir=train_img_dir, char_len=imgModel.char_len), batch_size=imgModel.batch_size, shuffle=True) train_dataloder_dataset_len = len(train_dataloder.dataset) if train_dataloder_dataset_len < imgModel.batch_size: # 图片数量少于 批次batch_size batch_acc_tatol = imgModel.char_len * train_dataloder_dataset_len else: # 图片数量 大于 批次batch_size batch_acc_tatol = imgModel.char_len * imgModel.batch_size for epoch in range(imgModel.epochs): if epoch_flag: break for batch_idx, (img, label) in enumerate( train_dataloder ): # batch_idx == len(train_dataloder.dataset) /batch_size if torch.cuda.is_available(): img, label = img.to(DEVICE), label.to(DEVICE) optimizer.zero_grad() output = imgModel(img) # print(output.size()) # torch.Size([batch_size512, 144]) label = label.view_as(output) loss = loss_fn(output, label) loss.backward() optimizer.step() if batch_idx % 100 == 0: # 每100个批次统计一次 batch_acc = imgModel._acc_count( output, label, batch_acc_tatol) # 当前批次batch准确度x% acces.append(batch_acc) per = sum(acces) / (len(acces) * 100) * 100. # 总准确度x% if per > imgModel.per: epoch_flag = True break print( 'time:{},第{}次训练,第{}批次:,loss_item:{},该批次内准确度:{}%,总准确度per:{}%' .format(datetime.now(), epoch, batch_idx, loss.item(), batch_acc, per)) model_file = 'l{}.pth'.format(imgModel.char_len) model_path = os.path.join(imgModel.model_dir, model_file) torch.save(imgModel.state_dict(), model_path)
def train(): x, y = load_data() model = ResNet() model = model.cuda() # Loss and Optimizer # criterion = nn.CrossEntropyLoss() criterion = nn.MultiLabelSoftMarginLoss() optimizer = torch.optim.Adam(model.parameters(), learning_rate) x = np.asarray(x, dtype=np.int32) y = np.asarray(y, dtype=np.int32) x_torch = torch.from_numpy(x) y_torch = torch.from_numpy(y) # 先转换成 torch 能识别的 Dataset torch_dataset = Data.TensorDataset(x_torch, y_torch) # 把 dataset 放入 DataLoader train_loader = Data.DataLoader( dataset=torch_dataset, # torch TensorDataset format batch_size=batch_size, # mini batch size shuffle=False, # 要不要打乱数据 (打乱比较好) num_workers=1, # 多线程来读数据 ) #traing the model for epoch in range(15): model.train() print("epoch {}".format(epoch)) for i, (images, labels) in enumerate(train_loader): images = Variable(images) #print(images.shape) labels = Variable(labels) # print(images.shape) # print(labels.shape) # #forward + backward + optimizer optimizer.zero_grad() images = images.float() labels = labels.long() if i == break_batch_tra: #进行到最后一个batch_size时, reshape 无法 break # 使用,直接跳过 labels = labels.reshape(batch_size, 1) labels = torch.zeros(batch_size, class_num).scatter_(1, labels, 1) ##one_hot 编码 outputs = model(images.cuda()) loss = criterion(outputs, labels.cuda()) loss.backward() optimizer.step() if 1: model.eval() total = 0 correct = 0 for i, (images, labels) in enumerate(train_loader): images = Variable(images) labels = Variable(labels) images = images.float() labels = labels.long() if i == break_batch_tra: #进行到最后一个batch_size时, reshape 无法 break # 使用,直接跳过 labels = labels.reshape(batch_size) outputs = model(images.cuda()) pred_data, pred = torch.max(outputs.data, 1) ##取出经过网络计算后的标签 test = outputs.data pred = pred.cpu().numpy() labels = labels.numpy() correct += (pred == labels).sum() total += np.size(labels, 0) accu = correct / total print("train correct is {}".format(correct)) print("train total is {}".format(total)) print("train accuracy is {}\n".format(accu)) # torch.save(model, './model/'+str(epoch)+'test-model.pkl') torch.save(model, './model/test-model.pkl') print("model has been written") return
######################################################################################## if __name__ == '__main__': print('%s: calling main function ... ' % os.path.basename(__file__)) # https://discuss.pytorch.org/t/print-autograd-graph/692/8 batch_size = 1 num_classes = 17 C, H, W = 3, 256, 256 inputs = torch.randn(batch_size, C, H, W) labels = torch.randn(batch_size, num_classes) in_shape = inputs.size()[1:] if 1: net = PyResNet34(in_shape=in_shape, num_classes=num_classes).cuda().train() x = inputs logits, probs = net.forward(x.cuda()) loss = nn.MultiLabelSoftMarginLoss()(logits, labels.cuda()) loss.backward() print(type(net)) print(net) print('probs') print(probs) #input('Press ENTER to continue.')
# 输出层 输出数量因设为想要的长度 此处是因为把验证码进行独热编码 self.fc2 = nn.Linear(1024, len(alphabet) * captcha_len) def forward(self, x): x = self.conv1(x) x = self.conv2(x) x = self.conv3(x) x = x.view(x.size(0), -1) x = self.fc1(x) x = F.relu(x) out = self.fc2(x) return out cnn = CNN() loss_func = nn.MultiLabelSoftMarginLoss() optimizer = torch.optim.Adam(cnn.parameters(), lr=0.001) def train(): # 开始训练 for i in range(5): for ind, (x, y, order) in enumerate(train_load): x, y = Variable(x), Variable(y) out_x = cnn(x) loss = loss_func(out_x, y) optimizer.zero_grad() loss.backward() optimizer.step() if ind % 50 == 0: print(f"Epoch: {i} | train los: {loss.data.numpy()}")