def main(): cuda = torch.cuda.is_available() anchor_transform = transforms.Compose([ transforms.RandomAffine(degrees=90, translate=(0.25, 0.25)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.CenterCrop(128), transforms.Resize(IMG_SIZE), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) train_transforms = transforms.Compose([ transforms.Resize(IMG_SIZE), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) # Let's use 12 while developing as it reduces the start time. dset_train = GeoTileDataset(TILE_FILE, transform=train_transforms, center_transform=anchor_transform) pd_files = dset_train.get_file_df() weights = pd_files.frequency train_sampler = WeightedRandomSampler(weights, len(dset_train)) # Should numworkers be 1? kwargs = {'num_workers': 8, 'pin_memory': True} if cuda else {} online_train_loader = DataLoader(dset_train, batch_size=BATCH_SIZE, sampler=train_sampler, **kwargs) model = Loc2Vec() if cuda: model.cuda() loss_fn = OnlineTripletLoss(MARGIN, HardestNegativeTripletSelector(MARGIN), SemihardNegativeTripletSelector(MARGIN), RandomNegativeTripletSelector(MARGIN)) optimizer = optim.Adam(model.parameters(), lr=1e-3) scheduler = lr_scheduler.StepLR(optimizer, 16, gamma=0.1, last_epoch=-1) # Mixed precision training model, optimizer = amp.initialize(model, optimizer, opt_level="O1") # if torch.cuda.device_count() > 1: # print("Let's use", torch.cuda.device_count(), "GPUs!") # model = nn.DataParallel(model) fit(online_train_loader, online_train_loader, model, loss_fn, optimizer, scheduler, N_EPOCHS, cuda, LOG_INTERVAL)
def main(): # Load and log experiment configuration config = load_config() logger.info(config) manual_seed = config.get('manual_seed', None) if manual_seed is not None: logger.info(f'Seed the RNG for all devices with {manual_seed}') torch.manual_seed(manual_seed) # see https://pytorch.org/docs/stable/notes/randomness.html torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # Create the model model = get_model(config) # use DataParallel if more than 1 GPU available device = config['device'] if torch.cuda.device_count() > 1 and not device.type == 'cpu': model = nn.DataParallel(model) logger.info(f'Using {torch.cuda.device_count()} GPUs for training') # put the model on GPUs logger.info(f"Sending the model to '{config['device']}'") model = model.to(device) # Log the number of learnable parameters logger.info( f'Number of learnable params {get_number_of_learnable_parameters(model)}' ) # Create loss criterion loss_criterion = get_loss_criterion(config) # Create evaluation metric eval_criterion = get_evaluation_metric(config) # Create data loaders loaders = get_train_loaders(config) # Create the optimizer optimizer = _create_optimizer(config, model) # Create learning rate adjustment strategy lr_scheduler = _create_lr_scheduler(config, optimizer) # Create model trainer trainer = _create_trainer(config, model=model, optimizer=optimizer, lr_scheduler=lr_scheduler, loss_criterion=loss_criterion, eval_criterion=eval_criterion, loaders=loaders) # Start training trainer.fit()
def objective(trial, **kwargs): # # Categorical parameter # optimizer = trial.suggest_categorical('optimizer', ['MomentumSGD', 'Adam']) # # Int parameter # num_layers = trial.suggest_int('num_layers', 1, 3) # # Uniform parameter dropout_prob = trial.suggest_uniform('dropout_prob', 0.0, 1.0) # # Loguniform parameter # learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e-2) # # Discrete-uniform parameter # drop_path_rate = trial.suggest_discrete_uniform('drop_path_rate', 0.0, 1.0, 0.01) print("dropout_prob: {}".format(dropout_prob)) kwargs.update(dropout_prob=dropout_prob) # Filenames for each trial must be made unique in order to access each checkpoint. # checkpoint_callback = pl.callbacks.ModelCheckpoint( # os.path.join(MODEL_DIR, "trial_{}".format(trial.number), "{epoch}"), monitor="val_acc" # ) # The default logger in PyTorch Lightning writes to event files to be consumed by # TensorBoard. We don't use any logger here as it requires us to implement several abstract # methods. Instead we setup a simple callback, that saves metrics from each validation step. metrics_callback = MetricsCallback() """ Main training routine specific for this project. """ # ------------------------ # 1 INIT a model and the LIGHTNING Experiment class # ------------------------ model = MLP(**kwargs) experiment = ImageClassificationExperiment(model=model, **kwargs) # ------------------------ # 2 INIT TRAINER # ------------------------ kwargs.update({ "logger": False, # "checkpoint_callback": checkpoint_callback, "callbacks": [metrics_callback], "early_stop_callback": PyTorchLightningPruningCallback(trial, monitor="val_loss") }) valid_kwargs = inspect.signature(pl.Trainer.__init__).parameters trainer_kwargs = dict( (name, kwargs[name]) for name in valid_kwargs if name in kwargs) trainer = pl.Trainer(**trainer_kwargs) # ------------------------ # 3 START TRAINING # ------------------------ trainer.fit(experiment) return metrics_callback.metrics[-1]["val_loss"].item()
def classification_baseline_exp(device='2', lr=1e-3, n_epochs=300, batch_size=128, log_interval=50): os.environ['CUDA_VISIBLE_DEVICES'] = str(device) standarizer = TaskbStandarizer(data_manager=Dcase18TaskbData()) mu, sigma = standarizer.load_mu_sigma(mode='train', device='a') train_dataset = DevSet(mode='train', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) test_dataset = DevSet(mode='test', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=1) test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=1) model = vggish_bn() model = model.cuda() loss_fn = torch.nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer=optimizer, step_size=30, gamma=0.5, last_epoch=-1) fit(train_loader=train_loader, val_loader=test_loader, model=model, loss_fn=loss_fn, optimizer=optimizer, scheduler=scheduler, n_epochs=n_epochs, log_interval=log_interval, metrics=[AccumulatedAccuracyMetric()])
def train_model(triplet_train_loader, triplet_val_loader, test_fun, margin, net_output, model_directory): target_embedding_net = TargetEmbeddingNet(300, net_output) triple_model = TripletNet(target_embedding_net, target_embedding_net) if cuda: triple_model.cuda() loss_fn = TripletLoss(margin) lr = 1e-4 optimizer = optim.Adam(triple_model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 1000, gamma=0.5, last_epoch=-1) n_epochs = 20 log_interval = 100 train_losses, val_losses, metrices = fit(triplet_train_loader, triplet_val_loader, triple_model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, callback_test=test_fun, keep_checkpoint_max=10, model_dir=model_directory) return metrices
# # Test : figure anchor, positive, negative images # img = triplet_image_datasets['val'][np.random.randint(10)] # tsf = transforms.ToPILImage() # for i in range(3): # tsf(img[i]).show() # print(img[i].size()) # # Test END use_gpu = torch.cuda.is_available() batch_size = 8 kwargs = {'num_workers': 1, 'pin_memory': True} if use_gpu else {} triplet_image_loaders = {x: DataLoader(triplet_image_datasets[x], batch_size=batch_size, shuffle=True if x == 'train' else False, **kwargs) for x in ['train', 'val']} # set up the network and training parameters margin = 1. lr = 1e-3 model = TripletNet() if use_gpu: model.cuda() loss_fn = nn.TripletMarginLoss(margin=margin) optimizer = optims.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, step_size=8, gamma=0.1, last_epoch=-1) num_epoch = 25 log_interval = 5 fit(triplet_image_loaders['train'], triplet_image_loaders['val'], model, loss_fn, optimizer, scheduler, num_epoch, use_gpu, log_interval)
# Set up the network and training parameters from losses import OnlineTripletLoss from utils import AllTripletSelector, HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch from metrics import AverageNonzeroTripletsMetric margin = 1. loss_fn = OnlineTripletLoss(margin, SemihardNegativeTripletSelector(margin)) lr = 1e-4 optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 100 log_interval = 50 print('start training') torch.autograd.set_detect_anomaly(True) fit(online_train_loader, online_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[AverageNonzeroTripletsMetric()]) print('done training') torch.save(model.state_dict(), './checkpoint/triplet_semi_hn_100_epochs_8_12.pth')
def main(): import torch from torch.optim import lr_scheduler import torch.optim as optim from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default fold_idx = 4 gamma = 1.0 margin = 1.0 DAsetting = False args = parser.parse_args() args.seed = 0 args.use_tensorboard = True args.save_model = True n_epochs = 100 folder_name = 'exp7_deep100' comment = 'w/bn fold_' + str(fold_idx) + '_g_' + str(gamma) + '_m_' + str( margin) use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True device = torch.device("cuda" if use_cuda else "cpu") #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False x_data, y_data = load_smt() #get subject number y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) #y_subj = np.concatenate([y_data,y_subj],axis=1) # For classification data valtype = 'subj' # if x_data.shape[2] != 60: # x_data = x_data[:,:,2:,:] # plt.imshow(x_data[1000,0,:,:]) # #subj - 0-27 train # train_subj1 = np.r_[0:27] # train_subj2 = np.r_[0:27]+54 # # test_subj = np.r_[27:54,54+27:108] #chidx = np.r_[7:11, 12:15, 17:21, 32:41] #오연조건 # chidx = np.r_[2:56, 60:62] # x_data = x_data[:,:,chidx,:] # For Domain adaptation setting if DAsetting: train_subj1 = np.r_[27:54] train_subj2 = np.r_[27:54] + 54 test_subj = np.r_[0:27, 54 + 0:54 + 27] trial_s = (0, 200) trial_t = (0, 200) trial_val = (0, 200) dataset_train1 = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj1, trial=trial_s) dataset_train2 = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) dataset_train = dataset_train1.__add__(dataset_train2) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train1 = TripletGiga(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj1, trial=trial_s) triplet_dataset_train2 = TripletGiga(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) triplet_dataset_train = triplet_dataset_train1.__add__( triplet_dataset_train2) triplet_dataset_test = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) else: #DG setting test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9, fold_idx * 9 + 54:fold_idx * 9 + 9 + 54] print('test subj:' + str(test_subj)) train_subj = np.setxor1d(np.r_[0:108], test_subj) trial_train = (0, 200) trial_val = (0, 200) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) triplet_dataset_test = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) ################################################################################################################### # make model for metric learning from networks import basenet, Deep4Net, EmbeddingDeep4CNN, EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN from losses import TripletLoss_dev2, TripLoss embedding_net = Deep4Net() print(embedding_net) model = TripletNet(embedding_net) #exp3-1 fc레이어 한층더 # model.fc = nn.Sequential( # nn.Linear(model.num_hidden,128), # nn.ReLU(), # nn.Dropout(), # nn.Linear(128,2) # ) if cuda: model.cuda() loss_fn = TripletLoss_dev2(margin, gamma).cuda() log_interval = 10 # ########################################################## # optimizer = optim.Adam(model.parameters(), lr=0.01) # scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=1, last_epoch=-1) # exp1 : 62ch 0~5fold까지 셋팅 # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.5, last_epoch=-1) #exp2 : 운동영역주변 20ch, train성능이 fit하지 않는 현상이 g=0.7,1.0 양족에서 모두 나타나서, 기존의 러닝레이트보다 강하게 줘보고 실험코자함 # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=1.0, last_epoch=-1) # # #exp4, exp5 optimizer = optim.SGD(model.parameters(), lr=0.005 / gamma, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.8, last_epoch=-1) #너무 빨리 떨구면 언더피팅하는듯 # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.8, last_epoch=-1) #너무 빨리 떨구면 언더피팅하는듯 # exp5 # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=0.5, last_epoch=-1) #model for validation evalmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) print('____________DANet____________') print(model) #save someting if (args.save_model): model_save_path = 'model/' + folder_name + '/' + comment + '/' if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if loging: fname = model_save_path + datetime.today().strftime( "%m_%d_%H_%M") + ".txt" f = open(fname, 'w') if args.use_tensorboard: writer = SummaryWriter(comment=comment) # load_model_path = 'C:\\Users\dk\PycharmProjects\giga_cnn\model\deep100_negsubj\\fold_0_g_0.7\danet_0.7_49.pt' #'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt' load_model_path = 'C:\\Users\dk\PycharmProjects\giga_cnn\model\exp6_basenet\\fold_0_g_0.6\danet_0.6_86.pt' load_model_path = None if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) for epochidx in range(n_epochs): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, epochidx, n_epochs, cuda, log_interval) print(epochidx) train_loss, train_score = eval(args, evalmodel, device, train_loader) eval_loss, eval_score = eval(args, evalmodel, device, test_loader) if args.use_tensorboard: writer.add_scalar('Train/Loss', np.mean(train_loss) / 100, epochidx) writer.add_scalar('Train/Acc', np.mean(train_score) / 100, epochidx) writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, epochidx) writer.add_scalar('Eval/Acc', np.mean(eval_score) / 100, epochidx) writer.close() if args.save_model: torch.save( model.state_dict(), model_save_path + 'danet_' + str(gamma) + '_' + str(epochidx) + '.pt')
n_samples=4) test_batch_sampler = myBalancedBatchSampler(test_dataset, n_classes=10, n_samples=4) kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} online_train_loader = torch.utils.data.DataLoader( train_dataset, batch_sampler=train_batch_sampler, **kwargs) online_test_loader = torch.utils.data.DataLoader( test_dataset, batch_sampler=test_batch_sampler, **kwargs) # Set up the network and training parameters from networks import EmbeddingNet, HardNet from losses import OnlineContrastiveLoss from utils import AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch margin = 1. embedding_net = HardNet() model = embedding_net if cuda: model.cuda() loss_fn = OnlineContrastiveLoss(margin, HardNegativePairSelector()) lr = 1e-3 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 100 log_interval = 10 fit(online_train_loader, online_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval)
**kwargs) # Set up the network and training parameters from networks import EmbeddingNet, TripletNet, HardNet from losses import TripletLoss margin = 1. embedding_net = HardNet() model = TripletNet(embedding_net) if cuda: model.cuda() loss_fn = TripletLoss(margin) lr = 1e-3 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 100 log_interval = 10 fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, log_file_path='triplet_hardnet_log.txt') # torch.save(model.state_dict(), 'triplet_params.pkl')
def main(): # print('fsafsdaf:', args.training_dataset, args.arch) print(">> Creating directory if it does not exist:\n>> '{}'".format( args.directory)) if not os.path.exists(args.directory): os.makedirs(args.directory) log_dir = os.path.join(args.directory, 'log') if not os.path.exists(log_dir): os.mkdir(log_dir) params = {'architecture': args.arch, 'pooling': args.pool} n_classes = args.n_classes n_samples = args.n_samples cuda = args.cuda input_size = args.image_size transform, transform_te, transform_label = init_transform(input_size) kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} online_train_loader, online_test_loader = init_data_loader( args.root, n_classes, n_samples, transform, transform_te, transform_label, kwargs) # Set up the network and training parameters model = init_network(params) parameters = [] # add feature parameters parameters.append({'params': model.features.parameters()}) if cuda: # print('model cuda:', cuda) model.cuda() pos_margin = 1.0 neg_margin = 0.3 # define optimizer if args.optimizer == 'sgd': optimizer = torch.optim.SGD(parameters, args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif args.optimizer == 'adam': optimizer = torch.optim.Adam(parameters, args.lr, weight_decay=args.weight_decay) metrics = [AverageNonzeroTripletsMetric()] if args.loss.startswith('OnlineContrastiveEucLoss'): loss_fn = OnlineContrastiveEucLoss(pos_margin, neg_margin, HardNegativePairSelector()) elif args.loss.startswith('OnlineContrastiveCosLoss'): loss_fn = OnlineContrastiveCosLoss(args.loss_margin) elif args.loss.startswith('OnlineTriplet'): loss_fn = OnlineTripletLoss( args.loss_margin, HardestNegativeTripletSelector(args.loss_margin)) exp_decay = math.exp(-0.01) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=exp_decay) writer = SummaryWriter(log_dir=log_dir) writer.add_graph(model.features, torch.ones([1, 3, 224, 224]).cuda()) fit(online_train_loader, online_test_loader, model, loss_fn, optimizer, scheduler, writer, metrics=metrics, args=args)
if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs model = nn.DataParallel(model) model.cuda() loss_fn = ContrastiveLoss(margin) lr = 1e-3 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 40 log_interval = 10 if not resume_from_pth: print("begin fit") record_history = fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) with open('./{}/record_history.pkl'.format(experiment_folder), 'wb') as pkl_file: pickle.dump(record_history, pkl_file) pkl_file.close() torch.save(model.state_dict(), trained_weight_file) else: with open('./{}/record_history.pkl'.format(experiment_folder), 'rb') as pkl_file: record_history = pickle.load(pkl_file) pkl_file.close() model.load_state_dict(torch.load(trained_weight_file))
optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4) if scheduler_name == 'StepLR': scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=8, gamma=0.1) elif scheduler_name == 'MultiStepLR': if use_augmentation: scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[20, 30], gamma=0.1) else: scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10, 15, 20], gamma=0.1) else: raise ValueError('Invalid scheduler') # Loss function loss_fn = BlendedLoss(loss_type, cross_entropy_flag) # Train (fine-tune) model fit(online_train_loader, model, loss_fn, optimizer, scheduler, nb_epoch, start_epoch = start_epoch, device=device, log_interval=log_interval, save_model_to=config.model_save_dir) elif config.mode == 'test': test_dataset_path = dataset_path #+ '/test/test_data' queries, db = test_data_loader(test_dataset_path) #model = load(file_path=config.model_to_test) result_dict = infer(model, queries, db) print(result_dict) from sklearn.metrics import recall_score, precision_score import numpy as np positives = [] k = 1
# 使用的网络和损失函数,以及数据选择的类 from networks import HardNet from losses import OnlineTripletLoss from utils import AllTripletSelector, HardestNegativeTripletSelector, RandomNegativeTripletSelector # 多种样本选择方式 # 参数配置 margin = 1. embedding_net = HardNet() model = embedding_net if cuda: model.cuda() loss_fn = OnlineTripletLoss(margin, HardestNegativeTripletSelector(margin)) lr = 1e-3 optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 100 log_interval = 10 # 训练 fit(online_train_loader, online_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[AverageNonzeroTripletsMetric()], log_file_path='onlinetriplet_hardnet_log.txt')
if os.path.isfile('./model/0922_checkpoint'): checkpoint = torch.load('./model/0922_checkpoint') model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) model.train() torch.save( { 'epoch': n_epochs, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss_fn, }, './model/0922_checkpoint') # %% fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) torch.save( { 'epoch': n_epochs, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss_fn, }, './model/0922_checkpoint') train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
n_epochs = args.epochs log_interval = args.log if args.checkpoint: checkpoint = torch.load('checkpoint.pth.tar') model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) scheduler.load_state_dict(checkpoint['scheduler_state_dict']) epoch = checkpoint['epoch'] print("Restarting training from checkpoint...") if args.classify: fit(train_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, start_epoch=epoch) else: fit(train_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[AverageNonzeroTripletsMetric()], start_epoch=epoch)
def classification_baseline_exp(device='2', ckpt_prefix='Run01', lr=1e-3, n_epochs=300, batch_size=128, log_interval=50, classify=True, log_level='INFO'): kwargs = locals() log_file = '{}/ckpt/classification_exp/{}.log'.format( ROOT_DIR, ckpt_prefix) if not os.path.exists(os.path.dirname(log_file)): os.makedirs(os.path.dirname(log_file)) logging.basicConfig(filename=log_file, level=getattr(logging, log_level.upper(), None)) logging.info(str(kwargs)) os.environ['CUDA_VISIBLE_DEVICES'] = str(device) standarizer = TaskbStandarizer(data_manager=Dcase18TaskbData()) mu, sigma = standarizer.load_mu_sigma(mode='train', device='a') train_dataset = DevSet(mode='train', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) test_dataset = DevSet(mode='test', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=1) test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=1) model = vggish_bn(classify) model = model.cuda() loss_fn = torch.nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer=optimizer, step_size=30, gamma=0.5, last_epoch=-1) train_hist = History(name='train/a') val_hist = History(name='val/a') ckpter = CheckPoint(model=model, optimizer=optimizer, path='{}/ckpt/classification_exp'.format(ckpt_prefix), prefix=ckpt_prefix, interval=1, save_num=1) fit(train_loader=train_loader, val_loader=test_loader, model=model, loss_fn=loss_fn, optimizer=optimizer, scheduler=scheduler, n_epochs=n_epochs, log_interval=log_interval, metrics=[AccumulatedAccuracyMetric()], train_hist=train_hist, val_hist=val_hist, ckpter=ckpter, logging=logging)
def main(): import torch from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default fold_idx = 0 gamma = 0.7 margin = 1.0 DAsetting = False args = parser.parse_args() args.seed = 0 args.use_tensorboard = True args.save_model = True n_epochs = 200 startepoch = 0 folder_name = 'exp2' comment = '22ch_deep4' + str(fold_idx) + '_g_' + str(gamma) + '_m_' + str( margin) use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True device = torch.device("cuda" if use_cuda else "cpu") gpuidx = 0 #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False x_data, y_data = load_bcic(fs=250) y_subj = np.zeros([9, 576]) for i in range(9): y_subj[i, :] = i * 2 y_subj = y_data.reshape(9, 576) + y_subj y_subj = y_subj.reshape(9 * 576) valtype = 'subj' # if x_data.shape[2] != 60: test_subj = np.r_[2] # train_subj = np.setdiff1d(bci_excellent, test_subj) # bci_excellent.sort() print('test subj:' + str(test_subj)) train_subj = np.setdiff1d(np.r_[0:9], test_subj) trial_train = (0, 576) trial_val = (0, 576) dataset_train = BCICDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) dataset_test = BCICDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletBCIC(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) triplet_dataset_test = TripletBCIC(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) ################################################################################################################### # make model for metric learning # from networks import DWConvNet, basenet,Deep4Net_origin, Deep4Net, Deep4NetWs, EmbeddingDeep4CNN,EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN, TripletNet_conv_clf import get_common as gc from losses import TripletLoss_dev2, TripLoss, ContrastiveLoss_dk dgnet = gc.dgnet(gamma=gamma) model = dgnet.model if cuda: model.cuda(device) loss_fn = dgnet.loss_fn.cuda(device) log_interval = 10 optimizer = dgnet.optimizer milestones = dgnet.milestones scheduler = dgnet.scheduler print('____________DANet____________') print(model) # # model_save_path = 'model/'+folder_name+'/'+comment+'/' # if (args.save_model): # if not os.path.isdir(model_save_path): # os.makedirs(model_save_path) # # if args.use_tensorboard: # writer = SummaryWriter(comment=comment) # writer.add_text('optimizer', str(optimizer)) # writer.add_text('scheduler', str(milestones)) # writer.add_text('model_save_path', model_save_path) # writer.add_text('model', str(model)) # writer.close() load_model_path = 'C:\\Users\dk\PycharmProjects\csdg_exp2\model\exp3_22\danet_0.7_99.pt' # if startepoch > 0: # load_model_path = model_save_path+'danet_'+str(gamma)+'_'+ str(startepoch) + '.pt' # model_save_path = model_save_path +'(cont)' # else: # load_model_path = None # if load_model_path is not None: # model.load_state_dict(torch.load(load_model_path,map_location='cuda:0')) # # for param in model.clf_net.parameters(): # param.requires_grad = False # # # model.clf_net.clf= nn.Sequential(nn.Linear(model.clf_net.embedding_net.num_hidden, 4), # nn.Dropout(), # nn.LogSoftmax(dim=1)).cuda() # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=0.0005) # optimizer = optim.Adam(model.parameters(),lr=0.01) for epochidx in range(1, 200): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, epochidx, n_epochs, cuda, gpuidx, log_interval) print(epochidx) # train(args, model.clf_net, device, train_loader, optimizer, scheduler) train_loss, train_score = eval(args, model.clf_net, device, train_loader) eval_loss, eval_score = eval(args, model.clf_net, device, test_loader)
def main(): import torch from torch.optim import lr_scheduler import torch.optim as optim from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False ismultitask = False loso = False if (args.save_model): model_save_path = 'model/triplet/' if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if loging: fname = model_save_path + datetime.today().strftime("%m_%d_%H_%M") + ".txt" f = open(fname, 'w') x_data, y_data = load_smt() y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) # nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51] valtype = 'sess' if valtype == 'loso': pass elif valtype == 'sess': from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN from losses import TripletLoss_dev2 # make model for metric learning margin = 1 embedding_net = EmbeddingShallowCNN() # clf_net = nn.Sequential(EmbeddingDeep4CNN(),nn.Linear(1000,2),nn.Dropout(p=1),nn.LogSoftmax(dim=1)) print(embedding_net) model = TripletNet(embedding_net) if cuda: model.cuda() loss_fn = TripletLoss_dev2(margin).cuda() n_epochs =1 log_interval = 10 load_model_path = None#'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt' model.fc = nn.Sequential(nn.Linear(1000, 2), nn.Dropout(p=0.5)) if load_model_path is not None: embedding_net.load_state_dict(torch.load(load_model_path)) # For classification dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs) #make model for classification newmodel = nn.Sequential(model.embedding_net, nn.Linear(1000, 2), nn.Dropout(p=0.5), nn.LogSoftmax(dim=1) ).to(device) print(newmodel) newmodel.to(device) optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9) # optimizer = optim.Adam(newmodel.parameters()) for epoch in range(0): train(args, newmodel, device, train_loader, optimizer, epoch) j_loss, j_score = eval(args, newmodel, device, test_loader) # For embedding triplet_dataset_train = TripletGiga(x=x_data, y=y_data,valtype=valtype, istrain=True, sess=1) triplet_train_loader = torch.utils.data.DataLoader(triplet_dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs) triplet_dataset_test = TripletGiga(x=x_data, y=y_data,valtype=valtype, istrain=False, sess=2, subj=-1) triplet_test_loader = torch.utils.data.DataLoader(triplet_dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=1, last_epoch=-1) from sklearn.pipeline import Pipeline from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.model_selection import ShuffleSplit, cross_val_score lda = LinearDiscriminantAnalysis() Testmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) # tempEmbeddingNet = nn.Sequential(model.embedding_net, # nn.Linear(1000,1000), # nn.Sigmoid()) # model = TripletNet(embedding_net) print(model) for temp in range(1, 30): # 10epoch마다 세이브 fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) j_loss, j_score = eval(args, Testmodel, device, train_loader) j_loss, j_score = eval(args, Testmodel, device, test_loader) torch.save(model.state_dict(), 'clf_' + str(temp) + '.pt') torch.save(model.state_dict(), 'shallowDG_150epoch_82acc' + str(temp) + '.pt') #for visualization dataset_train_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=True, sess=1) train_loader_subj = torch.utils.data.DataLoader(dataset_train_subj, batch_size=args.batch_size, shuffle=True, **kwargs) dataset_test_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=False, sess=2, subj=-1) test_loader_subj = torch.utils.data.DataLoader(dataset_test_subj, batch_size=args.batch_size, shuffle=False, **kwargs) # train_embeddings_tl, train_labels_tl = extract_features(train_loader_subj.dataset, model.embedding_net.convnet,0,100) # val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000) train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net,1000) val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net,1000) # # train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net.convnet[0], 1000) # val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000) # = train_labels_tl-train_labels_tl%2 # from torchvision import datasets, models, transforms # temp = model.embedding_net.children() # newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # for param in model.embedding_net.parameters(): # param.requires_grad = True #newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # from sklearn.manifold import TSNE tsne = TSNE(n_components=2,perplexity=30) #features = np.concatenate([train_embeddings_tl,val_embeddings_tl]) #val_labels_tl = val_labels_tl+2 #labels = np.concatenate([train_labels_tl,val_labels_tl]) train_tsne = tsne.fit_transform(train_embeddings_tl[0:2000]) # plot_embeddings(train_tsne,train_labels_tl[0:1000]) plot_features(train_tsne,train_labels_tl[0:2000]) plot_features3d(train_tsne,train_labels_tl[0:1000]%2) val_tsne = tsne.fit_transform(val_embeddings_tl) plot_embeddings(val_tsne, (val_labels_tl-108)-(val_labels_tl-108)%2) for param in model.embedding_net.parameters(): param.requires_grad = True #embedding_net2 = EmbeddingDeep4CNN() newmodel = nn.Sequential(model.embedding_net, nn.Linear(1000, 2), nn.Dropout(p=0.5), nn.LogSoftmax(dim=1), ).to(device) print(newmodel) #newmodel.fc_lr = nn.Linear(1000,2) newmodel.to(device) optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9) #optimizer = optim.Adam(newmodel.parameters()) for epoch in range(1, 100): train(args, newmodel, device, train_loader, optimizer, epoch) j_loss, j_score = eval(args, newmodel, device, test_loader) if args.save_model: torch.save(newmodel.state_dict(),'clf_83_8.pt') newmodel.load_state_dict(torch.load(load_model_path)) # Visualize feature maps activation = {} def get_activation(name): def hook(model, input, output): activation[name] = output.detach() return hook handle = model.embedding_net.convnet[0].register_forward_hook(get_activation('fc')) handle.remove() model.embedding_net.convnet[0]._forward_hooks.clear() train_loader.dataset with torch.no_grad(): model.eval() # num_ftrs = model.embedding_net.fc.out_features embeddings = np.zeros((len(train_loader.dataset), num_ftrs)) labels = np.zeros(len(train_loader.dataset)) k = 0 for images, target in train_loader: if cuda: images = images.cuda() embeddings[k:k + len(images)] = model.get_embedding(images).data.cpu().numpy() labels[k:k + len(images)] = target.numpy() k += len(images) features = SaveFeatures(model.embedding_net.convnet[0]) temp = features.features.data.cpu().numpy() del features.features torch.cuda.empty_cache() for images, target in train_loader: if cuda: images = images.cuda() output = model.embedding_net(images) activation = [] def get_activation(): def hook(model, input, output): activation.append(output) print(output) return hook act = activation['conv1'].squeeze() fig, axarr = plt.subplots(act.size(0)) for idx in range(act.size(0)): axarr[idx].imshow(act[idx]) actmap = [] def printnorm(self, input, output): # input is a tuple of packed inputs # output is a Tensor. output.data is the Tensor we are interested print('Inside ' + self.__class__.__name__ + ' forward') print('') print('input: ', type(input)) print('input[0]: ', type(input[0])) print('output: ', type(output)) print('') print('input size:', input[0].size()) print('output size:', output.data.size()) print('output norm:', output.data.norm()) return output.data model.embedding_net.convnet[0].register_forward_hook(printnorm) out = model(input) fig, axarr = plt.subplots(10) for idx in range(10): axarr[idx].imshow(temp[idx,1,:,:])
def main(): import torch from torch.optim import lr_scheduler import torch.optim as optim from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default fold_idx = 5 gamma = 1 DAsetting = False args = parser.parse_args() args.seed = 0 args.use_tensorboard = True use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False device = torch.device("cuda" if use_cuda else "cpu") #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False x_data, y_data = load_smt() #get subject number y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) # For classification data valtype = 'subj' # #subj - 0-27 train # train_subj1 = np.r_[0:27] # train_subj2 = np.r_[0:27]+54 # # test_subj = np.r_[27:54,54+27:108] # For Domain adaptation setting if DAsetting: train_subj1 = np.r_[27:54] train_subj2 = np.r_[27:54] + 54 test_subj = np.r_[0:27, 54 + 0:54 + 27] trial_s = (0, 200) trial_t = (0, 200) trial_val = (0, 200) dataset_train1 = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj1, trial=trial_s) dataset_train2 = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) dataset_train = dataset_train1.__add__(dataset_train2) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train1 = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj1, trial=trial_s) triplet_dataset_train2 = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) triplet_dataset_train = triplet_dataset_train1.__add__( triplet_dataset_train2) triplet_dataset_test = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) else: #DG setting test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9, fold_idx + 54:fold_idx + 9 + 54] train_subj = np.setxor1d(np.r_[0:108], test_subj) trial_train = (0, 200) trial_val = (0, 200) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) triplet_dataset_test = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) ################################################################################################################### # make model for metric learning from networks import Deep4Net, EmbeddingDeep4CNN, EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN from losses import TripletLoss_dev2 margin = 1.0 embedding_net = Deep4Net() print(embedding_net) model = TripletNet(embedding_net) if cuda: model.cuda() loss_fn = TripletLoss_dev2(margin, gamma).cuda() n_epochs = 1 log_interval = 10 # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.5, last_epoch=-1) optimizer = optim.Adam(model.parameters(), lr=0.01) scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=1, last_epoch=-1) #model for validation evalmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) print('____________DANet____________') print(model) comment = 'fold_' + str(fold_idx) + '_g_' + str(gamma) #save someting if (args.save_model): model_save_path = 'model/Deep4Net_(test0604)/' + comment + '/' if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if loging: fname = model_save_path + datetime.today().strftime( "%m_%d_%H_%M") + ".txt" f = open(fname, 'w') if args.use_tensorboard: writer = SummaryWriter(comment=comment) # load_model_path = model_save_path+'danet_0.7_49.pt' #'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt' load_model_path = None if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) for temp in range(1, 50): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) print(temp) train_loss, train_score = eval(args, evalmodel, device, train_loader) eval_loss, eval_score = eval(args, evalmodel, device, test_loader) if args.use_tensorboard: writer.add_scalar('Train/Loss', np.mean(train_loss) / 100, temp) writer.add_scalar('Train/Acc', np.mean(train_score) / 100, temp) writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, temp) writer.add_scalar('Eval/Acc', np.mean(eval_score) / 100, temp) writer.close() torch.save( model.state_dict(), model_save_path + 'danet_' + str(gamma) + '_' + str(temp) + '.pt')
def experiment(args): fold_idx = args.fold_idx startepoch = 0 folder_name = args.folder_name comment = args.comment torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True device = args.device #data load x_data, y_data = load_smt(fs=250) # get subject number y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) valtype = 'subj' test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9, fold_idx * 9 + 54:fold_idx * 9 + 9 + 54] print('test subj:' + str(test_subj)) train_subj = np.setdiff1d(np.r_[0:108], test_subj) trial_train = (0, 200) trial_val = (0, 200) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletGiga4(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) triplet_dataset_test = TripletGiga4(x=x_data, y=y_subj, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) #create model import get_common as gc dgnet = gc.dgnet(gamma=args.gamma, margin=args.margin) model = dgnet.model if cuda: model.cuda(device) loss_fn = dgnet.loss_fn if cuda and (loss_fn is not None): loss_fn.cuda(device) optimizer = dgnet.optimizer milestones = dgnet.milestones scheduler = dgnet.scheduler exp_comment = dgnet.exp_comment print(model) model_save_path = 'model/' + folder_name + '/' + comment + '/' if (args.save_model): if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if args.use_tensorboard: writer = SummaryWriter(log_dir=args.log_dir) writer.add_text('exp', exp_comment) writer.add_text('optimizer', str(optimizer)) writer.add_text('scheduler', str(milestones)) writer.add_text('model_save_path', model_save_path) writer.add_text('model', str(model)) model_save_path = writer.log_dir writer.close() if (args.save_model): if not os.path.isdir(model_save_path): os.makedirs(model_save_path) # if startepoch > 0: # # load_model_path = model_save_path + 'danet_' + str(args.gamma) + '_' + str(startepoch) + '.pt' # # model_save_path = model_save_path + '(cont)' # # else: load_model_path = None # load_model_path ="C:\\Users\\Starlab\\PycharmProjects\\csdg\\exp0719\\Sep25_01-59-47_DESKTOP-186GIONsubj_sim_g_0.7_m_1.0danet_0.7_99.pt" #기존모델 있으면 경로 if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) acc_all = np.empty((1, 18)) max = 0 for epochidx in range(1, args.epochs): print(epochidx) fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, epochidx, args.epochs, cuda, args.gpuidx, log_interval=10) train_loss, train_score = eval(args, model.clf_net, device, train_loader) eval_loss, eval_score = eval(args, model.clf_net, device, test_loader) eval_temp = np.array(eval_score) eval_temp = eval_temp.reshape(4, 18) #한폴드 9명 2세션 =18 acc = eval_temp.mean(0) / args.batch_size acc_m = acc.mean(0) if acc_m > max: max = acc_m print("highest acc : ", max) acc_all = np.vstack([acc_all, acc]) np.save('[DG]acc_all_' + str(args.fold_idx), acc_all) if args.use_tensorboard: for subj in range(18): writer.add_scalar( 'eachsubj/' + str(subj), np.sum(eval_score[subj * 2:subj * 2 + 2]) / 200, epochidx) writer.add_scalar('Train/Loss', np.mean(train_loss) / args.batch_size, epochidx) writer.add_scalar('Train/Acc', np.mean(train_score) / args.batch_size, epochidx) writer.add_scalar('Eval/Loss', np.mean(eval_loss) / args.batch_size, epochidx) writer.add_scalar('Eval/Acc', np.mean(eval_score) / args.batch_size, epochidx) writer.close() if args.save_model: torch.save( model.state_dict(), model_save_path + 'danet_' + str(args.gamma) + '_' + str(epochidx) + '.pt') acc_all = np.delete(acc_all, [0, 0], axis=0)
def main(): import torch from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default fold_idx = 4 gamma = 1.0 margin = 1.0 DAsetting = False args = parser.parse_args() args.seed = 0 args.use_tensorboard = True args.save_model = True n_epochs = 200 startepoch = 0 folder_name = 'exp1' comment = 'deep4' + str(fold_idx) + '_g_' + str(gamma) + '_m_' + str( margin) use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True device = torch.device("cuda:0" if use_cuda else "cpu") gpuidx = 0 #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False x_data, y_data = load_smt() x_data = x_data[:, :, :, 100:] #get subject number y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) #y_subj = np.concatenate([y_data,y_subj],axis=1) # plt.imshow(x_data[100,0,:,:]) # For classification data valtype = 'subj' # if x_data.shape[2] != 60: # x_data = x_data[:,:,2:,:] # plt.imshow(x_data[1000,0,:,:]) # #subj - 0-27 train # train_subj1 = np.r_[0:27] # train_subj2 = np.r_[0:27]+54 # # test_subj = np.r_[27:54,54+27:108] #chidx = np.r_[7:11, 12:15, 17:21, 32:41] #오연조건 # chidx = np.r_[2:56, 60:62] # x_data = x_data[:,:,chidx,:] # For Domain adaptation setting if DAsetting: # test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9, fold_idx * 9 + 54:fold_idx * 9 + 9 + 54] test_subj_id = 39 test_subj = np.r_[test_subj_id:test_subj_id + 1] train_subj1 = np.setxor1d(np.r_[0:108], test_subj) train_subj2 = test_subj n_targets = 60 trial_s = (0, 200) trial_t = (0, n_targets) trial_val = (n_targets, 200) # dataset_train1 = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True,subj=train_subj1,trial=trial_s) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) # dataset_train = dataset_train1.__add__(dataset_train2) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletGigaDA(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj_s=train_subj1, trial_s=trial_s, subj_t=train_subj2, trial_t=trial_t) # triplet_dataset_train2 = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj2, trial=trial_t) # triplet_dataset_train = triplet_dataset_train1.__add__(triplet_dataset_train2) triplet_dataset_test = TripletGigaDA(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj_s=train_subj1, trial_s=trial_s, subj_t=test_subj, trial_t=trial_val) else: #DG setting # test_subj = np.r_[fold_idx*9:fold_idx*9+9,fold_idx*9+54:fold_idx*9+9+54] # train_subj = test_subj # trial_train = (0, 30) # trial_val = (30, 200) # # bci_excellent = np.r_[43, 20, 27, 1, 28, 32, 35, 44, 36, 2] # bci_excellent = np.concatenate([bci_excellenth, bci_excellent + 54]) test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9, fold_idx * 9 + 54:fold_idx * 9 + 9 + 54] # train_subj = np.setdiff1d(bci_excellent, test_subj) # bci_excellent.sort() print('test subj:' + str(test_subj)) train_subj = np.setdiff1d(np.r_[0:108], test_subj) trial_train = (0, 200) trial_val = (0, 200) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) triplet_dataset_train = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj, trial=trial_train) # triplet_dataset_train2 = TripletGiga2(x=x_data[:,:,:,10:], y=y_subj, valtype=valtype, istrain=True, subj=train_subj, # trial=trial_train) # triplet_dataset_train = triplet_dataset_train1.__add__(triplet_dataset_train2) triplet_dataset_test = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=False, subj=test_subj, trial=trial_val) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) ################################################################################################################### # make model for metric learning # from networks import DWConvNet, basenet,Deep4Net_origin, Deep4Net, Deep4NetWs, EmbeddingDeep4CNN,EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN, TripletNet_conv_clf import get_common as gc from losses import TripletLoss_dev2, TripLoss, ContrastiveLoss_dk dgnet = gc.dgnet(gamma=gamma) model = dgnet.model if cuda: model.cuda(device) loss_fn = dgnet.loss_fn.cuda(device) log_interval = 10 optimizer = dgnet.optimizer milestones = dgnet.milestones scheduler = dgnet.scheduler print('____________DANet____________') print(model) model_save_path = 'model/' + folder_name + '/' + comment + '/' if (args.save_model): if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if args.use_tensorboard: writer = SummaryWriter(comment=comment) writer.add_text('optimizer', str(optimizer)) writer.add_text('scheduler', str(milestones)) writer.add_text('model_save_path', model_save_path) writer.add_text('model', str(model)) writer.close() if startepoch > 0: load_model_path = model_save_path + 'danet_' + str(gamma) + '_' + str( startepoch) + '.pt' model_save_path = model_save_path + '(cont)' else: load_model_path = None if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) for epochidx in range(1, 100): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, epochidx, n_epochs, cuda, gpuidx, log_interval) print(epochidx) train_loss, train_score = eval(args, model.clf_net, device, train_loader) eval_loss, eval_score = eval(args, model.clf_net, device, test_loader) if args.use_tensorboard: writer.add_scalar('Train/Loss', np.mean(train_loss) / args.batch_size, epochidx) writer.add_scalar('Train/Acc', np.mean(train_score) / args.batch_size, epochidx) writer.add_scalar('Eval/Loss', np.mean(eval_loss) / args.batch_size, epochidx) writer.add_scalar('Eval/Acc', np.mean(eval_score) / args.batch_size, epochidx) writer.close() if args.save_model: torch.save( model.state_dict(), model_save_path + 'danet_' + str(gamma) + '_' + str(epochidx) + '.pt')
def main(): import torch from torch.optim import lr_scheduler import torch.optim as optim from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False ismultitask = False loso = False if (args.save_model): model_save_path = 'model/triplet/' if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if loging: fname = model_save_path + datetime.today().strftime( "%m_%d_%H_%M") + ".txt" f = open(fname, 'w') x_data, y_data = load_smt() y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) # nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51] valtype = 'sess' if valtype == 'loso': for subj in range(0, 54): model = Deep4CNN(ismult=ismultitask).to(device) #model.load_state_dict(torch.load(model_save_path+ "J_" + str(subj) + 'basecnn.pt')) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) optimizer_fine = optim.SGD(model.parameters(), lr=0.005, momentum=args.momentum) dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1, subj=subj) train_loader = torch.utils.data.DataLoader( dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=subj) test_loader = torch.utils.data.DataLoader( dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs) # dataset_fine = GigaDataset_LOSO(x=x_data, y=y_data, fine=True, istrain=True, sess=2, subj=subj) # fine_loader = torch.utils.data.DataLoader(dataset_fine, batch_size=args.batch_size, shuffle=True, **kwargs) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) print("joint-train") #LOSO joint training j_loss, j_score = eval(args, model, device, test_loader) if epoch > 30: if (args.save_model): torch.save( model.state_dict(), model_save_path + "model_" + str(subj) + "_" + str(epoch) + '.pt') # #fine tuning # for epoch in range(1, 10): # train_mt(args, model, device, fine_loader, optimizer_fine, epoch) # # print("fine-tuning") # f_loss, f_score = eval(args, model, device, test_loader) if (args.save_model): torch.save(model.state_dict(), model_save_path + "F_" + str(subj) + 'basecnn.pt') if loging: f = open(fname, 'a') f.write( str(subj) + " " + "jl : " + str(j_loss) + " " + str(j_score) + '\n') f.close() elif valtype == 'sess': from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet from losses import TripletLoss_dev2 # make model for metric learning margin = 1 embedding_net = EmbeddingDeep4CNN() # clf_net = nn.Sequential(EmbeddingDeep4CNN(),nn.Linear(1000,2),nn.Dropout(p=1),nn.LogSoftmax(dim=1)) print(embedding_net) model = TripletNet(embedding_net) if cuda: model.cuda() loss_fn = TripletLoss_dev2(margin).cuda() n_epochs = 1 log_interval = 10 load_model_path = None #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt' model.fc = nn.Sequential(nn.Linear(1000, 2), nn.Dropout(p=0.5)) if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) # For classification dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1) train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs) #make model for classification newmodel = nn.Sequential(model.embedding_net, nn.Linear(1000, 2), nn.LogSoftmax(dim=1)).to(device) print(newmodel) newmodel.to(device) optimizer = optim.SGD(newmodel.parameters(), lr=0.01, momentum=0.9) # optimizer = optim.Adam(newmodel.parameters()) for epoch in range(0): train(args, newmodel, device, train_loader, optimizer, epoch) j_loss, j_score = eval(args, newmodel, device, test_loader) # For embedding triplet_dataset_train = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs) triplet_dataset_test = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=1, last_epoch=-1) from sklearn.pipeline import Pipeline from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.model_selection import ShuffleSplit, cross_val_score lda = LinearDiscriminantAnalysis() Testmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) print(Testmodel) for temp in range(1, 30): #10epoch마다 세이브 fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) j_loss, j_score = eval(args, Testmodel, device, test_loader) torch.save(model.state_dict(), 'clf_' + str(temp) + '.pt') Testmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) print(Testmodel) j_loss, j_score = eval(args, Testmodel, device, test_loader) dataset_train_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=True, sess=1) train_loader_subj = torch.utils.data.DataLoader( dataset_train_subj, batch_size=args.batch_size, shuffle=True, **kwargs) dataset_test_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=False, sess=2, subj=-1) test_loader_subj = torch.utils.data.DataLoader( dataset_test_subj, batch_size=args.batch_size, shuffle=False, **kwargs) train_embeddings_tl, train_labels_tl = extract_embeddings( train_loader_subj, model.embedding_net, 1000) val_embeddings_tl, val_labels_tl = extract_embeddings( test_loader_subj, model.embedding_net, 1000) train_labels_tl_subj = train_labels_tl - train_labels_tl % 2 # from torchvision import datasets, models, transforms # temp = model.embedding_net.children() # newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # for param in model.embedding_net.parameters(): # param.requires_grad = True #newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # from sklearn.manifold import TSNE tsne = TSNE(n_components=2, perplexity=30) #features = np.concatenate([train_embeddings_tl,val_embeddings_tl]) #val_labels_tl = val_labels_tl+2 #labels = np.concatenate([train_labels_tl,val_labels_tl]) train_tsne = tsne.fit_transform(train_embeddings_tl) plot_embeddings(train_tsne, train_labels_tl % 2) val_tsne = tsne.fit_transform(val_embeddings_tl) plot_embeddings(val_tsne, val_labels_tl % 2) for param in model.embedding_net.parameters(): param.requires_grad = True #embedding_net2 = EmbeddingDeep4CNN() newmodel = nn.Sequential( model.embedding_net, nn.Linear(1000, 2), nn.Dropout(p=0.5), nn.LogSoftmax(dim=1), ).to(device) print(newmodel) #newmodel.fc_lr = nn.Linear(1000,2) newmodel.to(device) optimizer = optim.SGD(newmodel.parameters(), lr=0.01, momentum=0.9) #optimizer = optim.Adam(newmodel.parameters()) for epoch in range(1, 100): train(args, newmodel, device, train_loader, optimizer, epoch) j_loss, j_score = eval(args, newmodel, device, test_loader) if args.save_model: torch.save(newmodel.state_dict(), 'clf_83_8.pt') newmodel.load_state_dict(torch.load('clf_83_8.pt'))
def hard_triplet_baseline_exp(device='3', ckpt_prefix='Run01', lr=1e-3, n_epochs=300, n_classes=10, n_samples=12, margin=0.3, log_interval=50, log_level="INFO"): """ :param device: :param lr: :param n_epochs: :param n_classes: :param n_samples: :return: """ kwargs = locals() log_file = '{}/ckpt/hard_triplet_baseline_exp/{}.log'.format( ROOT_DIR, ckpt_prefix) if not os.path.exists(os.path.dirname(log_file)): os.makedirs(os.path.dirname(log_file)) logging.basicConfig(filename=log_file, level=getattr(logging, log_level.upper(), None)) logging.info(str(kwargs)) os.environ['CUDA_VISIBLE_DEVICES'] = str(device) # get the mean and std of dataset train/a standarizer = TaskbStandarizer(data_manager=Dcase18TaskbData()) mu, sigma = standarizer.load_mu_sigma(mode='train', device='a') # get the normalized train dataset train_dataset = DevSet(mode='train', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) test_dataset = DevSet(mode='test', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) train_batch_sampler = BalanceBatchSampler(dataset=train_dataset, n_classes=n_classes, n_samples=n_samples) train_batch_loader = DataLoader(dataset=train_dataset, batch_sampler=train_batch_sampler, num_workers=1) test_batch_sampler = BalanceBatchSampler(dataset=test_dataset, n_classes=n_classes, n_samples=n_samples) test_batch_loader = DataLoader(dataset=test_dataset, batch_sampler=test_batch_sampler, num_workers=1) model = networks.embedding_net_shallow() model = model.cuda() loss_fn = RandomHardTripletLoss( margin=margin, triplet_selector=RandomNegativeTripletSelector(margin=margin)) optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer=optimizer, step_size=30, gamma=0.5) fit(train_loader=train_batch_loader, val_loader=test_batch_loader, model=model, loss_fn=loss_fn, optimizer=optimizer, scheduler=scheduler, n_epochs=n_epochs, log_interval=log_interval, metrics=[AverageNoneZeroTripletsMetric()]) verification(model=model) train_embedding_tl, train_labels_tl = extract_embeddings( train_batch_loader, model, 64) # utils.plot_embeddings(embeddings=train_embedding_tl, targets=train_labels_tl, title='train set') test_embedding_tl, test_labels_tl = extract_embeddings( test_batch_loader, model, 64) # utils.plot_embeddings(embeddings=test_embedding_tl, targets=test_labels_tl, title='test set') model2 = networks.classifier() model2 = model2.cuda() loss_fn2 = nn.CrossEntropyLoss() optimizer2 = optim.Adam(model2.parameters(), lr=lr) scheduler2 = lr_scheduler.StepLR(optimizer=optimizer2, step_size=30, gamma=0.5) train_dataset2 = DatasetWrapper(data=train_embedding_tl, labels=train_labels_tl, transform=ToTensor()) test_dataset2 = DatasetWrapper(data=test_embedding_tl, labels=test_labels_tl, transform=ToTensor()) train_loader2 = DataLoader(dataset=train_dataset2, batch_size=128, shuffle=True, num_workers=1) test_loader2 = DataLoader(dataset=test_dataset2, batch_size=128, shuffle=False, num_workers=1) train_hist = History(name='train/a') val_hist = History(name='val/a') ckpter = CheckPoint( model=model, optimizer=optimizer, path='{}/ckpt/hard_triplet_baseline_exp'.format(ROOT_DIR), prefix=ckpt_prefix, interval=1, save_num=1) fit(train_loader=train_loader2, val_loader=test_loader2, model=model2, loss_fn=loss_fn2, optimizer=optimizer2, scheduler=scheduler2, n_epochs=n_epochs, log_interval=log_interval, metrics=[AccumulatedAccuracyMetric()], train_hist=train_hist, val_hist=val_hist, ckpter=ckpter, logging=logging)
i_triplet_train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, **kwargs) i_triplet_val_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=validation_sampler, **kwargs) # Set up the network and training parameters text_embedding_net = TextEmbeddingNet(dim=output_embedding_size) if feature_mode == 'resnet152': image_embedding_net = Resnet152EmbeddingNet(dim=output_embedding_size) elif feature_mode == 'resnet18': image_embedding_net = Resnet18EmbeddingNet(dim=output_embedding_size) model = IntermodalTripletNet(image_embedding_net, text_embedding_net) if cuda: model.cuda() loss_fn = InterTripletLoss(margin) optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = optim.lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) log_interval = 100 fit(i_triplet_train_loader, i_triplet_val_loader, dataset.intermodal_triplet_batch_sampler, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) pickle.dump(model, open('pickles/models/entire_nuswide_model_14.p', 'wb'))
def main(): import torch from torch.optim import lr_scheduler import torch.optim as optim from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=100, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default writer = SummaryWriter() args = parser.parse_args() args.seed = 0 use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False device = torch.device("cuda" if use_cuda else "cpu") #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False if (args.save_model): model_save_path = 'model/DANet(Deep4Net100)_test/' if not os.path.isdir(model_save_path): os.makedirs(model_save_path) if loging: fname = model_save_path + datetime.today().strftime( "%m_%d_%H_%M") + ".txt" f = open(fname, 'w') x_data, y_data = load_smt() #get subject number y_subj = np.zeros([108, 200]) for i in range(108): y_subj[i, :] = i * 2 y_subj = y_data.reshape(108, 200) + y_subj y_subj = y_subj.reshape(21600) # nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51] valtype = 'sess' if valtype == 'loso': pass elif valtype == 'sess': from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN from losses import TripletLoss_dev2 # make model for metric learning margin = 1.0 gamma = 0.7 embedding_net = EmbeddingDeep4CNN() print(embedding_net) model = TripletNet(embedding_net) #model.fc = nn.Linear(embedding_net.num_hidden,2) if cuda: model.cuda() loss_fn = TripletLoss_dev2(margin, gamma).cuda() n_epochs = 1 log_interval = 10 #load_model_path = model_save_path+'dgnet1.027.pt' #'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt' load_model_path = None if load_model_path is not None: model.load_state_dict(torch.load(load_model_path)) # For classification data dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1) train_loader = torch.utils.data.DataLoader( dataset_train, batch_size=args.batch_size, shuffle=True, ) dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1) test_loader = torch.utils.data.DataLoader( dataset_test, batch_size=args.batch_size, shuffle=False, ) # For Domain adaptation triplet_dataset_train = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_dataset_test = TripletGiga(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=0.5, last_epoch=-1) #model for validation evalmodel = nn.Sequential(model.embedding_net, model.fc, nn.LogSoftmax(dim=1)).to(device) print('____________DANet____________') print(model) # # from torch.utils.tensorboard import SummaryWriter # writer = SummaryWriter() # images, labels = next(iter(train_loader)) # import torchvision # #grid = torchvision.utils.make_grid(images) # writer.add_images('images',images) # # writer.add_embedding(metadata=train_embeddings_tl) # # writer.add_embedding(metadata = val_embeddings_tl) # writer.close() for temp in range(1, 50): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) train_loss, j_score = eval(args, evalmodel, device, train_loader) eval_loss, j_score = eval(args, evalmodel, device, test_loader) writer.add_scalar('Train/Loss', np.mean(train_loss) / 100, temp) writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, temp) writer.close() np.mean(train_loss) torch.save( model.state_dict(), model_save_path + 'dgnet_' + str(gamma) + '_' + str(temp) + '.pt') #for visualization dataset_train_subj = GigaDataset(x=x_data, y=y_subj, valtype='subj', istrain=True, sess=1, subj=np.r_[0:10]) train_loader_subj = torch.utils.data.DataLoader( dataset_train_subj, batch_size=args.batch_size, shuffle=True, ) dataset_test_subj = GigaDataset(x=x_data, y=y_subj, valtype='sess', istrain=False, sess=2, subj=-1) test_loader_subj = torch.utils.data.DataLoader( dataset_test_subj, batch_size=args.batch_size, shuffle=False, ) # train_embeddings_tl, train_labels_tl = extract_features(train_loader_subj.dataset, model.embedding_net.convnet,0,100) # val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000) train_embeddings_tl, train_labels_tl = extract_embeddings( train_loader_subj, model.embedding_net, model.embedding_net.num_hidden) val_embeddings_tl, val_labels_tl = extract_embeddings( test_loader_subj, model.embedding_net, model.embedding_net.num_hidden) # # train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net.convnet[0], 1000) # val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000) # = train_labels_tl-train_labels_tl%2 # from torchvision import datasets, models, transforms # temp = model.embedding_net.children() # newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # for param in model.embedding_net.parameters(): # param.requires_grad = True #newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:])) # from sklearn.manifold import TSNE tsne = TSNE(n_components=2, perplexity=30) #features = np.concatenate([train_embeddings_tl,val_embeddings_tl]) #val_labels_tl = val_labels_tl+2 #labels = np.concatenate([train_labels_tl,val_labels_tl]) train_tsne = tsne.fit_transform(train_embeddings_tl) plot_features(train_tsne, train_labels_tl) plot_features(train_tsne, train_labels_tl % 2) for i in range(0, 10): plot_features(train_tsne[i * 200:(i + 1) * 200], train_labels_tl[i * 200:(i + 1) * 200]) plot_features3d(train_tsne, train_labels_tl) val_tsne = tsne.fit_transform(val_embeddings_tl) plot_features(val_tsne, val_labels_tl - 108) plot_features(val_tsne, val_labels_tl % 2) plot_features3d(val_tsne, val_labels_tl % 2) plot_embeddings(val_tsne, (val_labels_tl - 108) - (val_labels_tl - 108) % 2) for param in model.embedding_net.parameters(): param.requires_grad = True #embedding_net2 = EmbeddingDeep4CNN() newmodel = nn.Sequential( model.embedding_net, nn.Linear(1000, 2), nn.Dropout(p=0.5), nn.LogSoftmax(dim=1), ).to(device) print(newmodel) #newmodel.fc_lr = nn.Linear(1000,2) newmodel.to(device) optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9) #optimizer = optim.Adam(newmodel.parameters()) for epoch in range(1, 100): train(args, newmodel, device, train_loader, optimizer, epoch) j_loss, j_score = eval(args, newmodel, device, test_loader) if args.save_model: torch.save(newmodel.state_dict(), 'clf_83_8.pt') newmodel.load_state_dict(torch.load(load_model_path))
embedding_net = EmbeddingNet() model = ClassificationNet(embedding_net, n_classes=n_classes) if cuda: model.cuda() loss_fn = torch.nn.NLLLoss() lr = 1e-2 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 20 log_interval = 50 fit(train_loader, test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[AccumulatedAccuracyMetric()]) train_embeddings_baseline, train_labels_baseline = extract_embeddings( train_loader, model) plot_embeddings(train_embeddings_baseline, train_labels_baseline) val_embeddings_baseline, val_labels_baseline = extract_embeddings( test_loader, model) plot_embeddings(val_embeddings_baseline, val_labels_baseline) # Set up data loaders from datasets import SiameseMNIST
def main(i): import torch from torch.autograd import Variable from trainer import fit import numpy as np cuda = torch.cuda.is_available() # Training settings parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=10, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') # Writer will output to ./runs/ directory by default fold_idx = 4 gamma = 0.7 margin = 0.5 DAsetting = False args = parser.parse_args() args.seed = 0 args.use_tensorboard = True args.save_model = True n_epochs = 200 startepoch = 0 folder_name = 'exp1' comment = 'deep4' + str(fold_idx) + '_g_' + str(gamma) + '_m_' + str( margin) use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True device = torch.device("cuda:0" if use_cuda else "cpu") gpuidx = 0 #kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} from datetime import datetime import os loging = False x_data, y_data, sizes = load_speech(path='', data_type='i') x_data = x_data[:, :, :, 50:] #get subject number x_data2, y_data2, sizes2 = load_speech(path='', data_type='o') x_data2 = x_data2[:, :, :, 50:] y_subj = np.ones_like(y_data) test_subj = i # 0:15, 1:19. 2: 20,// 3:27, 4:17, 5:25, 8:12 # test_idx = np.r_[sum(sizes[0:test_subj]):sum(sizes[0:test_subj + 1])] # train_idx = np.setdiff1d(np.r_[0:y_subj.shape[0]], test_idx) # bci_excellent.sort() print('test subj:' + str(test_subj)) use_split = True from sklearn.model_selection import train_test_split if use_split: test_idx = np.r_[sum(sizes[0:test_subj]):sum(sizes[0:test_subj + 1])] test_idx_overt = np.r_[sum(sizes2[0:test_subj] ):sum(sizes2[0:test_subj + 1])] # 전체 일경우 # test_idx = np.r_[0:y_data.shape[0]] # test_idx_overt = np.r_[0:y_data2.shape[0]] x_data_imagery = x_data[test_idx] y_data_imagery = y_data[test_idx] x_data_overt = x_data2[test_idx_overt] y_data_overt = y_data2[test_idx_overt] X_train, X_test, y_train, y_test = train_test_split(x_data_imagery, y_data_imagery, test_size=0.3, random_state=42) X_train = np.concatenate([X_train, x_data_overt]) y_train = np.concatenate([y_train, y_data_overt]) # dataset = SpeechDataset(x=x_data, y=y_data, idx=np.r_[0:y_subj.shape[0]], train=True) dataset_train = SpeechDataset(x=X_train, y=y_train, idx=None, train=True) dataset_test = SpeechDataset(x=X_test, y=y_test, idx=None, train=False) triplet_dataset_train = TripletSpeech(dataset_train, subset=False) triplet_dataset_test = TripletSpeech(dataset_test, subset=False) else: test_idx = np.r_[sum(sizes[0:test_subj]):sum(sizes[0:test_subj + 1])] train_idx = np.setdiff1d(np.r_[0:y_subj.shape[0]], test_idx) dataset_train = SpeechDataset(x=x_data, y=y_data, idx=train_idx, train=True) dataset_test = SpeechDataset(x=x_data, y=y_data, idx=test_idx, train=False) triplet_dataset_train = TripletSpeech(dataset_train) triplet_dataset_test = TripletSpeech(dataset_test) # triplet_dataset_train = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=True, subj=train_subj, # trial=trial_train) # # triplet_dataset_train2 = TripletGiga2(x=x_data[:,:,:,10:], y=y_subj, valtype=valtype, istrain=True, subj=train_subj, # # trial=trial_train) # # triplet_dataset_train = triplet_dataset_train1.__add__(triplet_dataset_train2) # # triplet_dataset_test = TripletGiga2(x=x_data, y=y_subj, valtype=valtype, istrain=False, subj=test_subj, # trial=trial_val) # train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) triplet_train_loader = torch.utils.data.DataLoader( triplet_dataset_train, batch_size=args.batch_size, shuffle=True) triplet_test_loader = torch.utils.data.DataLoader( triplet_dataset_test, batch_size=args.batch_size, shuffle=False) ################################################################################################################### # make model for metric learning # from networks import DWConvNet, basenet,Deep4Net_origin, Deep4Net, Deep4NetWs, EmbeddingDeep4CNN,EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN, TripletNet_conv_clf import get_common as gc # from losses import TripletLoss_dev2, TripLoss, ContrastiveLoss_dk margin = 1 dgnet = gc.dgnet(gamma=gamma, margin=margin) model = dgnet.model if cuda: model.cuda(device) loss_fn = dgnet.loss_fn if cuda and (loss_fn is not None): loss_fn.cuda(device) optimizer = dgnet.optimizer milestones = dgnet.milestones scheduler = dgnet.scheduler exp_comment = dgnet.exp_comment print('____________DANet____________') print(model) model_save_path = 'model/' + folder_name + '/' + comment + '/' # if (args.save_model): # if not os.path.isdir(model_save_path): # os.makedirs(model_save_path) # # if startepoch > 0: # load_model_path = model_save_path+'danet_'+str(gamma)+'_'+ str(startepoch) + '.pt' # model_save_path = model_save_path +'(cont)' # else: # load_model_path = None # if load_model_path is not None: # model.load_state_dict(torch.load(load_model_path)) log_interval = 10 maxacc = 0 for epochidx in range(1, 100): fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, epochidx, n_epochs, cuda, gpuidx, log_interval) print(epochidx) train_loss, train_score, tacc = eval(args, model.clf_net, device, train_loader) eval_loss, eval_score, acc = eval(args, model.clf_net, device, test_loader) if acc > maxacc: maxacc = acc # if args.use_tensorboard: # writer.add_scalar('Train/Loss', np.mean(train_loss)/args.batch_size, epochidx) # writer.add_scalar('Train/Acc', np.mean(train_score)/args.batch_size, epochidx) # writer.add_scalar('Eval/Loss', np.mean(eval_loss)/args.batch_size, epochidx) # writer.add_scalar('Eval/Acc', np.mean(eval_score)/args.batch_size, epochidx) # writer.close() # if args.save_model: # torch.save(model.state_dict(), model_save_path + 'danet_'+str(gamma)+'_'+ str(epochidx) + '.pt') return maxacc
print('datasets loaded') train_loader = torch.utils.data.DataLoader(train_set, batch_size=cfg.data.imgs_per_gpu, shuffle=True) query_loader = torch.utils.data.DataLoader(query_set, batch_size=cfg.data.imgs_per_gpu, shuffle=True) print('dataloader created') # Build model backbone_model = Vgg16L2(num_dim=128) model = TripletNet(backbone_model) model.cuda() print('model built') margin = 1. loss_fn = TripletLoss(margin) lr = 1e-4 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 20 log_interval = 500 print('start training') fit(train_loader, query_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval) print('done training') torch.save(model.state_dict(), './checkpoint/triplet_vanilla_20_epochs_1e-4.pth')
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) embedding_net = EmbeddingNet() model = ClassificationNet(embedding_net, n_classes=n_classes) if cuda: model.cuda() loss_fn = torch.nn.NLLLoss() lr = 1e-2 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 1 log_interval = 50 fit(train_loader, test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[AccumulatedAccuracyMetric()])