def fe_extr_warm_start(self, sp_feature_ext, writer=None): # dloader = DataLoader(MultiDiscSpGraphDsetBalanced(length=self.args.fe_warmup_iterations * 10), batch_size=10, # shuffle=True, pin_memory=True) dloader = DataLoader( MultiDiscSpGraphDset(length=self.args.fe_warmup_iterations * 10), batch_size=10, shuffle=True, pin_memory=True) criterion = ContrastiveLoss(delta_var=0.5, delta_dist=1.5) optimizer = torch.optim.Adam(sp_feature_ext.parameters(), lr=2e-3) for i, (data, gt) in enumerate(dloader): data, gt = data.to(sp_feature_ext.device), gt.to( sp_feature_ext.device) pred = sp_feature_ext(data) l2_reg = None if self.args.l2_reg_params_weight != 0: for W in list(sp_feature_ext.parameters()): if l2_reg is None: l2_reg = W.norm(2) else: l2_reg = l2_reg + W.norm(2) if l2_reg is None: l2_reg = 0 loss = criterion(pred, gt) + l2_reg * self.args.l2_reg_params_weight optimizer.zero_grad() loss.backward() optimizer.step() if writer is not None: writer.add_scalar("loss/fe_warm_start", loss.item(), self.writer_idx_warmup_loss) self.writer_idx_warmup_loss += 1
def __init__(self, widgets_dataset: ImageDataset, train, margin=1.0): self.widgets_dataset = widgets_dataset self.classes = ["negative", "positive"] self.c = 2 self.loss_func = ContrastiveLoss(margin) self.path = widgets_dataset.path self.train = train self.transform = self.widgets_dataset._tfms self._data_paths, self._labels = list(zip(*self.widgets_dataset._data)) self._labels = torch.LongTensor(self._labels) self.labels_set = set(self._labels.numpy()) self.label_to_indices = { label: np.where(self._labels.numpy() == label)[0] for label in self.labels_set } if not self.train: random_state = np.random.RandomState(29) positive_pairs = [[ i, random_state.choice( self.label_to_indices[self._labels[i].item()]), 1, ] for i in range(0, len(self._data_paths), 2)] negative_pairs = [[ i, random_state.choice(self.label_to_indices[np.random.choice( list(self.labels_set - set([self._labels[i].item()])))]), 0, ] for i in range(1, len(self._data_paths), 2)] self.test_pairs = positive_pairs + negative_pairs
def siamese_learner(data, pretrained_model_class, emsize=128, margin=1.0, callback_fns=None): meta = cnn_config(pretrained_model_class) model = create_cnn_model(pretrained_model_class, emsize) model = SiameseNet(model) learn = Learner(data, model, loss_func=ContrastiveLoss(margin), callback_fns=callback_fns) learn.split(meta["split"](model.embedding_net)) apply_init(model.embedding_net[1], nn.init.kaiming_normal_) return learn
def fe_extr_warm_start(self, sp_feature_ext, writer=None): dataloader = DataLoader(MultiDiscSpGraphDset(length=100), batch_size=10, shuffle=True, pin_memory=True) criterion = ContrastiveLoss(delta_var=0.5, delta_dist=1.5) optimizer = torch.optim.Adam(sp_feature_ext.parameters()) for i, (data, gt) in enumerate(dataloader): data, gt = data.to(sp_feature_ext.device), gt.to(sp_feature_ext.device) pred = sp_feature_ext(data[:,0,:,:].unsqueeze(1)) loss = criterion(pred, gt) optimizer.zero_grad() loss.backward() optimizer.step() if writer is not None: writer.add_scalar("loss/fe_warm_start", loss.item(), self.writer_idx_warmup_loss) self.writer_idx_warmup_loss += 1
def __init__(self, cfg, global_count, global_writer_loss_count, global_writer_quality_count, action_stats_count, global_writer_count, save_dir): super(AgentSacTrainer_test_sg_global, self).__init__() self.cfg = cfg self.global_count = global_count self.global_writer_loss_count = global_writer_loss_count self.global_writer_quality_count = global_writer_quality_count self.action_stats_count = action_stats_count self.global_writer_count = global_writer_count self.contr_trpl_loss = ContrastiveTripletLoss(delta_var=0.5) self.contr_loss = ContrastiveLoss(delta_var=0.5, delta_dist=1.5) self.memory = TransitionData_ts(capacity=self.cfg.trainer.t_max) # self.eps = self.args.init_epsilon self.save_dir = save_dir
def parse_parameters(params, optimizer, lr, loss, schedule=None): optimizer = optimizer.lower() if 'adam' in optimizer: optimizer = optim.Adam(lr=lr, params=params, weight_decay=1e-5) elif 'sgd' in optimizer: optimizer = optim.SGD(lr=lr, momentum=0.9, params=params) else: print('optimizer {} not yet supported'.format(optimizer)) scheduler = None if schedule is not None: scheduler = optim.lr_scheduler.ExponentialLR(optimizer, math.exp(schedule)) loss = loss.lower() if 'cont' in loss: criterion = ContrastiveLoss() elif 'trip' in loss: criterion = TripletLoss() else: print('loss {} not supported'.format(loss)) return (optimizer, criterion, scheduler)
def __init__(self, args): super(SiameseGAN, self).__init__() self.epoch = args.epoch self.sample_num = args.batch_size self.batch_size = args.batch_size self.save_dir = args.save_dir self.result_dir = args.result_dir self.dataset = args.dataset self.log_dir = args.log_dir self.gpu_mode = args.gpu_mode self.model_name = args.gan_type self.input_size = args.input_size self.z_dim = 100 self.c = 0.01 # clipping value self.n_critic = 1 # the number of iterations of the critic per generator iteration self.margin = 2.0 self.margin_decay = 0.5 self.transform = transforms.Compose([ transforms.Resize((self.input_size, self.input_size)), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) self.data_loader = torch.utils.data.DataLoader( datasets.MNIST('./data', train=True, download=True, transform=self.transform), batch_size=self.batch_size, shuffle=True) # Download Mnist def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv') != -1: m.weight.data.normal_(0.0, 0.02) elif classname.find('BatchNorm') != -1: m.weight.data.normal_(1.0, 0.02) m.bias.data.fill_(0) self.G = generator(64) self.G.apply(weights_init) self.D = discriminator(64) self.D.apply(weights_init) self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2)) self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2)) if self.gpu_mode: self.G.to('cuda:0') self.D.to('cuda:0') self.CLloss = ContrastiveLoss() print('---------- Networks architecture -------------') utils.print_network(self.G) utils.print_network(self.D) print('-----------------------------------------------') self.sample_z_ = torch.rand( (self.batch_size, self.z_dim)).view(-1, 100, 1, 1) if self.gpu_mode: self.sample_z_ = self.sample_z_.to('cuda:0')
model_pretrained.load_state_dict(torch.load('resnet50-19c8e357.pth')) # weights of imagenet num_ftrs = model_pretrained.fc.in_features # fc_in model_pretrained.fc = nn.Linear(num_ftrs, num_category) # modify fc_out num_cls = model_pretrained.fc.out_features # fc_out model = mymodel.myresnet50(model_pretrained, num_cls) # load my model model_dis = mymodel.Discriminator() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) model_dis.to(device) from trainer import set_parameter_requires_grad set_parameter_requires_grad(model) params_to_update = [] for name,param in model.named_parameters(): if param.requires_grad == True: params_to_update.append(param) from losses import ContrastiveLoss margin = 1. criterion_sia = ContrastiveLoss(margin) criterion_ft = nn.CrossEntropyLoss() criterion_dis = nn.BCELoss() optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9) optimizer_dis = optim.Adam(model_dis.parameters(), lr=0.002, betas=(0.5, 0.999)) scheduler_ft = lr_scheduler.StepLR(optimizer_ft, step_size=10, gamma=0.1) from trainer import adv_fit adv_fit(siamese_train_loader, model, model_dis, saving_model_name, device, test_loader, criterion_sia, criterion_ft, criterion_dis, optimizer_ft, optimizer_dis, scheduler_ft, num_epochs)
def main(): torch.manual_seed(args.seed) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices use_gpu = torch.cuda.is_available() if args.use_cpu: use_gpu = False if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) if use_gpu: print("Currently using GPU {}".format(args.gpu_devices)) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) else: print("Currently using CPU (GPU is highly recommended)") #print("Initializing dataset {}".format(args.dataset)) # dataset = data_manager.init_dataset(name=args.dataset) transform_train = T.Compose([ T.Random2DTranslation(args.height, args.width), T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) transform_train_p = T.Compose([ T.Random2DTranslation(256, 128), T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) transform_test = T.Compose([ T.Resize((args.height, args.width)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) transform_test_p = T.Compose([ T.Resize((256, 128)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) train_file = 'data/cuhk_train.pkl' test_file = 'data/cuhk_test.pkl' gallery_file = 'data/cuhk_gallery.pkl' data_root = args.data_root dataset_train = CUHKGroup(train_file, data_root, True, transform_train, transform_train_p) dataset_test = CUHKGroup(test_file, data_root, False, transform_test, transform_test_p) dataset_query = CUHKGroup(test_file, data_root, False, transform_test, transform_test_p) dataset_gallery = CUHKGroup(gallery_file, data_root, False, transform_test, transform_test_p) pin_memory = True if use_gpu else False if args.xent_only: trainloader = DataLoader( dataset_train, batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=pin_memory, drop_last=True, ) else: trainloader = DataLoader( dataset_train, sampler=RandomIdentitySampler(dataset_train, num_instances=args.num_instances), batch_size=args.train_batch, num_workers=args.workers, pin_memory=pin_memory, drop_last=True, ) queryloader = DataLoader( dataset_test, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=pin_memory, drop_last=False, ) querygalleryloader = DataLoader( dataset_query, batch_size=args.gallery_batch, shuffle=False, num_workers=args.workers, pin_memory=pin_memory, drop_last=True, ) galleryloader = DataLoader( dataset_gallery, batch_size=args.gallery_batch, shuffle=False, num_workers=args.workers, pin_memory=pin_memory, drop_last=True, ) print("Initializing model: {}".format(args.arch)) if args.xent_only: # model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent'}) model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent'}) else: # model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent', 'htri'}) model = models.init_model( name=args.arch, num_classes=dataset_train.num_train_gids, num_person_classes=dataset_train.num_train_pids, loss={'xent', 'htri'}) #criterion_xent = CrossEntropyLabelSmooth(num_classes=dataset_train.num_train_gids, use_gpu=use_gpu) #criterion_xent_person = CrossEntropyLabelSmooth(num_classes=dataset_train.num_train_pids, use_gpu=use_gpu) if os.path.exists(args.pretrained_model): print("Loading checkpoint from '{}'".format(args.pretrained_model)) checkpoint = torch.load(args.pretrained_model) model_dict = model.state_dict() pretrain_dict = checkpoint['state_dict'] pretrain_dict = { k: v for k, v in pretrain_dict.items() if k in model_dict } model_dict.update(pretrain_dict) model.load_state_dict(model_dict) criterion_xent = nn.CrossEntropyLoss(ignore_index=-1) criterion_xent_person = nn.CrossEntropyLoss(ignore_index=-1) criterion_htri = TripletLoss(margin=args.margin) criterion_pair = ContrastiveLoss(margin=args.margin) criterion_htri_filter = TripletLossFilter(margin=args.margin) criterion_permutation = PermutationLoss() optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.weight_decay) if args.stepsize > 0: if args.warmup: scheduler = WarmupMultiStepLR(optimizer, [200, 400, 600]) else: scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma) start_epoch = args.start_epoch if use_gpu: model = nn.DataParallel(model).cuda() if args.evaluate: print("Evaluate only") test_gcn_person_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu) #test_gcn_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu) #test_gcn(model, queryloader, galleryloader, args.pool, use_gpu) #test(model, queryloader, galleryloader, args.pool, use_gpu) return start_time = time.time() best_rank1 = -np.inf for epoch in range(start_epoch, args.max_epoch): #print("==> Epoch {}/{} lr:{}".format(epoch + 1, args.max_epoch, scheduler.get_lr()[0])) train_gcn(model, criterion_xent, criterion_xent_person, criterion_pair, criterion_htri_filter, criterion_htri, criterion_permutation, optimizer, trainloader, use_gpu) #train(model, criterion_xent, criterion_htri, optimizer, trainloader, use_gpu) if args.stepsize > 0: scheduler.step() if args.eval_step > 0 and (epoch + 1) % args.eval_step == 0 or ( epoch + 1) == args.max_epoch: print("==> Test") rank1 = test_gcn_person_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu) #rank1 = test_gcn(model, queryloader, galleryloader, args.pool, use_gpu=False) #rank1 = test(model, queryloader, galleryloader, args.pool, use_gpu) is_best = rank1 > best_rank1 if is_best: best_rank1 = rank1 if use_gpu: state_dict = model.module.state_dict() else: state_dict = model.state_dict() save_checkpoint( { 'state_dict': state_dict, 'rank1': rank1, 'epoch': epoch, }, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch + 1) + '.pth.tar')) elapsed = round(time.time() - start_time) elapsed = str(datetime.timedelta(seconds=elapsed)) print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
#real_imgs = np.expand_dims(real_imgs,axis=3) #gen_imgs = np.expand_dims(gen_imgs,axis=3) sec_per = np.zeros((batch_size,3,opt.temporal_length,25,1),dtype=np.float32) sec_per = torch.from_numpy(sec_per).cuda() #real_imgs_hcn = torch.unsqueeze(real_imgs,4) gen_imgs_hcn = torch.unsqueeze(fake_imgs,4) #real_imgs_hcn = torch.cat((real_imgs_hcn,sec_per),-1) gen_imgs_hcn = torch.cat((gen_imgs_hcn,sec_per),-1) #print (real_imgs.shape) #print (gen_imgs.shape) #print (sec_per.shape) out2 = hcn_model(gen_imgs_hcn) if opt.feat_loss == 'contr': contrastive_loss = ContrastiveLoss(margin=2.0) contrastive_loss = contrastive_loss(out1,out2,intersec_labels) feat_loss = contrastive_loss elif opt.feat_loss == 'triplet': #triplet_loss = TripletLoss(margin=2.0) #triplet_loss = triplet_loss(anchor=out2,positive=out1,negative=out3) triplet_loss = triplet_loss_torch(anchor=out2,positive=out1,negative=out3) #print ("Triplet loss manual",triplet_loss) #print ("Triplet loss torch",triplet_loss_torch) feat_loss = triplet_loss elif opt.feat_loss == 'cosineembed': pos = Variable(FloatTensor(batch_size, 1).fill_(1.0), requires_grad=False) neg = Variable(FloatTensor(batch_size, 1).fill_(-1.0), requires_grad=False) cosine_loss_pos = cosine_embedding_loss(out2,out1,pos) cosine_loss_neg = cosine_embedding_loss(out2,out3,neg)
def run_experiment(sim_type, network_type, dataset, mus, latent_dims, batch_size=64, keys=None, composite_labels=None, margin=1, dropout_val=0.2, num_iters=20000, normalize=True, feature_extractor='hog'): """Overarching function that runs an experiment. Calls _experiment for each combination of mu and latent dim. Args: sim_type (str): Simulation type. Accepts either "pairwise" or "triplet" network_type (str): Network type. Accepts either "embenc" or "vae" dataset (str): Dataset name. Accepts "mnist", "cifar10" and "imagenet" mus (list): list of mu values for which experiment is to be run. latent_dims (list): list of latent dimensions for which experiment is to be run. batch_size (int, optional): batch size from which pairs/triples are generated. Defaults to 64. keys (dict, optional): Mapping from numeric labels to string label names. Defaults to None. composite_labels (dict, optional): Maps current labels to composite labels. Defaults to None. margin (int, optional): Margin to be used for metric loss. Defaults to 1. dropout_val (float, optional): dropout value for networks. Defaults to 0.2. num_iters (int, optional): number of iterations. Defaults to 20000. Returns: dict: mapping from (mu, latent_dim) to results for _experiment on those params. """ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("loading data...", end='') trainX, trainY, testX, testY, trainLabels, testLabels = load_data( dataset, composite_labels=composite_labels, normalize=normalize, feature_extractor=feature_extractor) print("done.") # instantiate the generator and the networks based on type. print("initializing data generators and losses...", end='') input_dim = trainX.shape[-1] if sim_type == 'pairwise': datagen = PairGenerator(trainX, trainY, batch_size) metric_loss = ContrastiveLoss( margin) if network_type != 'vae' else PairwiseVAELoss(margin) elif sim_type == 'triplet': datagen = TripletGenerator(trainX, trainY, batch_size) metric_loss = TripletLoss( margin) if network_type != 'vae' else TripletVAELoss(margin) reconstruction_loss = reconstruction_error print("done.") all_results = {} for mu in mus: for latent_dim in latent_dims: network = initialize_network(sim_type, network_type, input_dim, latent_dim, dropout_val, device=device) results = _experiment(network, latent_dim, datagen, metric_loss, reconstruction_loss, trainX, trainY, testX, testY, keys, mu, dataset, trainLabels, testLabels, num_iters=num_iters, batch_size=batch_size, device=device) all_results[(mu, latent_dim)] = results return all_results
for row_num in range(num_samples): img1 = ax[row_num, 0].imshow(get_img(inputs1.cpu().data[row_num])) img2 = ax[row_num, 1].imshow(get_img(inputs2.cpu().data[row_num])) ax[row_num, 0].xaxis.set_visible(False) ax[row_num, 0].yaxis.set_visible(False) ax[row_num, 1].xaxis.set_visible(False) ax[row_num, 1].yaxis.set_visible(False) ax[row_num, 0].set_title("Target: {}".format(target.cpu().data[row_num].item())) # for j in range(inputs1.size()[0]): # images_so_far += 1 # ax = plt.subplot(num_samples, 2, images_so_far) # ax.axis('off') # ax.set_title('target: {}'.format(target[j])) # imshow(inputs1.cpu().data[j]) if images_so_far == num_samples: model.train(mode=was_training) return plt.show() model.train(mode=was_training) dloaders = {'train':train_dataloader, 'valid':valid_dataloader} siameseNet = SiameseNet().cuda() criterion = ContrastiveLoss(margin=10) optimizer = optim.Adam(siameseNet.parameters(), lr=0.001) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) model_ft = train_model(dloaders, siameseNet, criterion, optimizer, exp_lr_scheduler, num_epochs=50) # visualize_model(model_ft)
def fe_extr_warm_start(self, sp_feature_ext, writer=None): # dloader = DataLoader(MultiDiscSpGraphDsetBalanced(length=self.args.fe_warmup_iterations * 10), batch_size=10, # shuffle=True, pin_memory=True) dloader = DataLoader(MultiDiscSpGraphDset( length=self.args.fe_warmup_iterations * 10, less=True, no_suppix=False), batch_size=1, shuffle=True, pin_memory=True) contrastive_l = ContrastiveLoss(delta_var=0.5, delta_dist=1.5) dice = GraphDiceLoss() small_lcf = nn.Sequential( nn.Linear(sp_feature_ext.n_embedding_channels, 256), nn.Linear(256, 512), nn.Linear(512, 1024), nn.Linear(1024, 256), nn.Linear(256, 1), ) small_lcf.cuda(device=sp_feature_ext.device) optimizer = torch.optim.Adam(sp_feature_ext.parameters(), lr=1e-3) for i, (data, node_labeling, gt_pix, gt_edges, edge_index) in enumerate(dloader): data, node_labeling, gt_pix, gt_edges, edge_index = data.to(sp_feature_ext.device), \ node_labeling.squeeze().to(sp_feature_ext.device), \ gt_pix.to(sp_feature_ext.device), \ gt_edges.squeeze().to(sp_feature_ext.device), \ edge_index.squeeze().to(sp_feature_ext.device) node_labeling = node_labeling.squeeze() stacked_superpixels = [ node_labeling == n for n in node_labeling.unique() ] sp_indices = [sp.nonzero() for sp in stacked_superpixels] edge_features, pred_embeddings, side_loss = sp_feature_ext( data, edge_index, torch.zeros_like(gt_edges, dtype=torch.float), sp_indices) pred_edge_weights = small_lcf(edge_features) l2_reg = None if self.args.l2_reg_params_weight != 0: for W in list(sp_feature_ext.parameters()): if l2_reg is None: l2_reg = W.norm(2) else: l2_reg = l2_reg + W.norm(2) if l2_reg is None: l2_reg = 0 loss_pix = contrastive_l(pred_embeddings.unsqueeze(0), gt_pix) loss_edge = dice(pred_edge_weights.squeeze(), gt_edges.squeeze()) loss = loss_pix + self.args.weight_edge_loss * loss_edge + \ self.args.weight_side_loss * side_loss + l2_reg * self.args.l2_reg_params_weight optimizer.zero_grad() loss.backward() optimizer.step() if writer is not None: writer.add_scalar("loss/fe_warm_start/ttl", loss.item(), self.writer_idx_warmup_loss) writer.add_scalar("loss/fe_warm_start/pix_embeddings", loss_pix.item(), self.writer_idx_warmup_loss) writer.add_scalar("loss/fe_warm_start/edge_embeddings", loss_edge.item(), self.writer_idx_warmup_loss) writer.add_scalar("loss/fe_warm_start/gcn_sideloss", side_loss.item(), self.writer_idx_warmup_loss) self.writer_idx_warmup_loss += 1
def main(): opt = parse_option() set_seed(opt.seed) print("Starting Training ..... \n\n") # create dataloader train_partition = 'trainval' if opt.use_trainval else 'train' train_loader, val_loader, n_cls = get_train_loaders(opt, train_partition) opt.n_cls = n_cls # CE loss criterion_cls = nn.CrossEntropyLoss() # create model model = ContrastResNet(opt, n_cls) # training parameters params = [{'params': model.encoder.parameters(), 'lr': opt.learning_rate}] # spatial contrastive loss if opt.spatial_cont_loss: attention = AttentionSimilarity(hidden_size=model.encoder.feat_dim, inner_size=opt.feat_dim, aggregation=opt.aggregation) params = params + [{'params': attention.parameters(), 'lr': opt.learning_rate}] criterion_contrast_spatial = ContrastiveLoss(temperature=opt.temperature_s) else: attention, criterion_contrast_spatial = None, None # global contrastive loss if opt.global_cont_loss: params = params + [{'params': model.head.parameters(), 'lr': opt.learning_rate}] criterion_contrast = ContrastiveLoss(temperature=opt.temperature_g) else: criterion_contrast = None # optimizer if opt.adam: optimizer = torch.optim.Adam(params, lr=opt.learning_rate, weight_decay=opt.weight_decay) else: optimizer = optim.SGD(params, lr=opt.learning_rate, momentum=opt.momentum, weight_decay=opt.weight_decay) # Set cuda params if opt.syncBN: model = apex.parallel.convert_syncbn_model(model) if torch.cuda.is_available(): model = model.cuda() criterion_cls = criterion_cls.cuda() if opt.global_cont_loss: criterion_contrast = criterion_contrast.cuda() if opt.spatial_cont_loss: attention = attention.cuda() criterion_contrast_spatial = criterion_contrast_spatial.cuda() cudnn.benchmark = True if opt.n_gpu > 1: model = nn.DataParallel(model) # tensorboard if opt.use_tb: logger = tb_logger.Logger(logdir=opt.tb_folder, flush_secs=2) # set cosine annealing scheduler if opt.cosine: eta_min = opt.learning_rate * (opt.lr_decay_rate ** 3) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, opt.epochs, eta_min, -1) # training routine for epoch in range(1, opt.epochs + 1): if opt.cosine: scheduler.step() else: adjust_learning_rate(epoch, opt, optimizer) time1 = time.time() train_loss = train(epoch, train_loader, model, criterion_cls, criterion_contrast, criterion_contrast_spatial, attention, optimizer, opt) time2 = time.time() print('epoch: {}, total time: {:.2f}, train loss: {:.3f}'.format(epoch, time2 - time1, train_loss)) if opt.use_tb and (epoch % opt.tb_freq) == 0: logger.log_value('train_loss', train_loss, epoch) # regular saving if epoch % opt.save_freq == 0: print('==> Saving...') state = { 'opt': opt, 'model': model.state_dict() if opt.n_gpu <= 1 else model.module.state_dict(), 'attention': attention.state_dict() if opt.spatial_similarity else None } save_file = os.path.join(opt.save_folder, 'ckpt_epoch_{epoch}.pth'.format(epoch=epoch)) torch.save(state, save_file) # save the last model state = { 'opt': opt, 'model': model.state_dict() if opt.n_gpu <= 1 else model.module.state_dict(), 'attention': attention.state_dict() if opt.spatial_cont_loss else None } save_file = os.path.join(opt.save_folder, '{}_last.pth'.format(opt.model)) torch.save(state, save_file)
classifier = models.ClassifierPro() #encoder = models.Encoder() encoder = tmodels.resnet18(pretrained=True) #encoder = tmodels.inception_v3(pretrained=True) #encoder.aux_logits=False encoder.fc = nn.Sequential() discriminator = models.DCDPro() #discriminator = models.DCDPro(input_features=128) classifier.to(device) encoder.to(device) discriminator.to(device) loss_fn = torch.nn.CrossEntropyLoss() loss_fn2 = ContrastiveLoss() ##quitar loss_fn3 = SpecLoss() ##quitar # ----------------------------------------------------------------------------- ## etapa 1: entrenar g y h print("||||| Stage 1 |||||") optimizer = torch.optim.Adam(list(encoder.parameters()) + list(classifier.parameters()), lr=0.0001) scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[50], gamma=0.1) #herb std-mean #tensor([0.0808, 0.0895, 0.1141]) #tensor([0.7410, 0.7141, 0.6500]) #photo std-mean #tensor([0.1399, 0.1464, 0.1392]) #tensor([0.2974, 0.3233, 0.2370])
embedding_dim = int(sys.argv[2]) m = float(sys.argv[1]) dir = "data/Gaussian/" TRAIN_CSV = dir + "training" TEST_CSV = dir + "points" WRITE_CSV = dir + "reducedPoints" gpus = 0 n_epoch = 20 num_threads = 8 test_size = 0.2 params = {'batch_size': 10240, 'shuffle': True} loss_fn = ContrastiveLoss(m) # Load training set train_df = pd.read_csv(TRAIN_CSV, delimiter=',', encoding="utf-8-sig") training_samples = train_df[['P1', 'P2']] training_labels = train_df[[ 'distance', 'cutoff', 'thisCluster', 'otherCluster' ]] input_dim = len(training_samples['P1'][0].split()) training_samples = training_samples.values training_labels = training_labels.values cuda = torch.cuda.is_available()
return stage_1_base_params, stage_1_classifier_params stage_1 = False stage_2 = True if stage_1: margin = 1. embedding_net = ft_net_dense() model = SiameseNet(embedding_net) # model = Sggnn(SiameseNet(embedding_net)) if use_gpu: model.cuda() # save_whole_network(model, 'best') # exit() loss_fn = ContrastiveLoss(margin) # loss_fn = SigmoidLoss() # loss_fn = nn.CrossEntropyLoss() lr = 1e-3 step = 8 # stage_1_base_params, stage_1_classifier_params = stage_1_params(model) # optimizer = optim.Adam([ # {'params': stage_1_base_params, 'lr': 1 * lr}, # {'params': stage_1_classifier_params, 'lr': 1 * lr}, # ]) optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, step, gamma=0.1, last_epoch=-1)
def train(epoch, model, criterion, optimizer, trainloader, use_contraloss, use_gpu): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() model.train() end = time.time() for batch_idx, (imgs, pids, _) in enumerate(trainloader): if use_gpu: imgs, pids = imgs.cuda(), pids.cuda() # measure data loading time data_time.update(time.time() - end) if use_contraloss: contrastiveloss = ContrastiveLoss() y_g, y_cp, y_p, f_g, f_cp = model(imgs) loss_contra = contrastiveloss(f_g, f_cp, 1) else: y_g, y_cp, y_p = model(imgs) loss_g = criterion(y_g, pids) loss_cp = criterion(y_cp, pids) part = {} sm = nn.Softmax(dim=1) num_part = 6 for i in range(num_part): part[i] = y_p[i] # score = sm(part[0]) + sm(part[1]) +sm(part[2]) + sm(part[3]) +sm(part[4]) +sm(part[5]) # _, preds = torch.max(score.data, 1) loss_p = criterion(part[0], pids) for i in range(num_part - 1): loss_p += criterion(part[i + 1], pids) # else: # loss = criterion(outputs, pids) if use_contraloss: loss = loss_g + loss_cp + loss_p / num_part + loss_contra else: loss = loss_g + loss_cp + loss_p / num_part optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() losses.update(loss.item(), pids.size(0)) if (batch_idx + 1) % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch + 1, batch_idx + 1, len(trainloader), batch_time=batch_time, data_time=data_time, loss=losses))
return combined_model if __name__ == "__main__": args = parse_args() if torch.cuda.is_available(): device = torch.device("cuda") torch.cuda.set_device(args.local_rank) else: device = torch.device("cpu") criterion_unsupervised = MMDLoss(1) criterion_supervised = ContrastiveLoss(margin=0.2, measure="cosine", reduction=args.reduction, max_violation=True) print("initializing dataloaders") print('=' * 30 + '\n') coco_data = CoCoDataset(args.coco, 'train', 224) coco_val = CoCoDataset(args.coco, 'val', 224) semart_data = SemArtDataset(semart_root=args.semart, split="train", desired_length=len(coco_data), img_input_size=224) semart_val = SemArtDataset(semart_root=args.semart, split="val", img_input_size=224) wpi_data = SemArtDataset(
def run_experiments(): ###################################EXPERIMENT_1############################################################## ''' DESCRIPTION Training and testing set both contain all the recordings. 80-20 split random state = 42 ''' ''' ID: 5924295 ''' list_IDs = [] train_list_IDs = [] test_list_IDs = [] y = [] IDs = [1, 2, 3, 4, 5] list_IDs, y = separate_data_by_mic_id_train(IDs) print(len(list_IDs), 'all') train_list_IDs, test_list_IDs, y_train, y_test = train_test_split( list_IDs, y, test_size=0.2, random_state=42) # 100 print(train_list_IDs, 'train') print(test_list_IDs, 'test') ######HYPERPARAMETERS############################################# num_epochs = 10 num_classes = 2 learning_rate = 1e-5 batch_size = 1 contrastive_loss_margin = 3.0 triplet_loss_margin = 1.0 #weight_counter = 1 ################################################################# siamese_training_set = SiameseDataset(train_list_IDs, y_train, True) #TripletDataset siamese_train_loader = torch.utils.data.DataLoader(dataset=siamese_training_set, batch_size=batch_size, shuffle=True) siamese_test_set = SiameseDataset(test_list_IDs, y_test, False) # TripletDataset siamese_test_loader = torch.utils.data.DataLoader(dataset=siamese_test_set, batch_size=batch_size, shuffle=True) embedding_net = EmbeddingNet(num_classes) model = SiameseNet(embedding_net) # TripletNet print('outside model') if cuda: model.cuda() # Loss and optimizer #criterion = TripletLoss(triplet_loss_margin) criterion = ContrastiveLoss(contrastive_loss_margin) optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # training train_mode=True print('starting training') fit(siamese_train_loader, siamese_test_loader, model, criterion, optimizer, num_epochs, use_cuda, train_mode) PATH='/mnt/nfs/scratch1/dghose/audio_siamese/siamese_weights_log_{}_{}_{}_{}_{}.pth'.format(num_epochs,num_classes,learning_rate,batch_size,contrastive_loss_margin) # unique names torch.save(model.state_dict(), PATH) model.load_state_dict(torch.load(PATH)) # Test train_mode=False fit(siamese_train_loader, siamese_test_loader, model, criterion, optimizer, num_epochs, use_cuda, train_mode) '''
def main(csv_file): data = pd.read_csv(csv_file) print( "The number of classes {} \n The number of Images in the Dataset is {}" .format(len(set(data['class'])), data.shape[0])) histogram = {} for name in data['class']: if name in histogram: histogram[name] += 1 else: histogram[name] = 1 class_max = getKeysByValue(histogram, max(histogram.values())) class_min = getKeysByValue(histogram, min(histogram.values())) print("classes with maximum examples {} and the number of examples {}". format(class_max, max(histogram.values()))) print("\n") print("number of classes with one example {}".format(len(class_min))) siamese_dataset = SiameseNetworkDataset( csv_file='meta.csv', transform=transforms.Compose([ transforms.Resize((224, 224)), transforms.RandomHorizontalFlip(), transforms.ToTensor() ])) validation_split = 0.8 train_size = int(validation_split * len(siamese_dataset)) test_size = len(siamese_dataset) - train_size train_dataset, test_dataset = torch.utils.data.random_split( siamese_dataset, [train_size, test_size]) train_dataloader = DataLoader(train_dataset, shuffle=True, num_workers=Config.number_of_workers, batch_size=Config.batch_size) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #device = torch.device("cpu") net = SiameseNetwork().to(device) criterion = ContrastiveLoss() optimizer = optim.Adam(net.parameters(), lr=0.0005) counter = [] loss_history = [] iteration_number = 0 print("Number of iterations in an epoch:", data.shape[0] // Config.batch_size) net.train() for epoch in range(0, Config.number_epochs): t = perf_counter() t1 = perf_counter() for i, data_i in enumerate(train_dataloader, 0): img0, img1, label = data_i img0, img1, label = img0.to(device), img1.to(device), label.to( device) optimizer.zero_grad() output1, output2 = net(img0, img1) loss_contrastive = criterion(output1, output2, label) loss_contrastive.backward() optimizer.step() if i % 500 == 0: print("Epoch number {}\n Current loss {}\n".format( epoch, loss_contrastive.item())) iteration_number += 500 counter.append(iteration_number) loss_history.append(loss_contrastive.item()) filename = 'siamese_temp_' + str(epoch) + '.pt' torch.save(net, filename) t2 = perf_counter() print("Elapsed time taken for 500 iteration in epoch", t2 - t1) t1 = t2 # Condition to break each epoch if i >= data.shape[0] // Config.batch_size: #data.shape[0]: t3 = perf_counter() print("Time taken for each epoch", t3 - t) break save_plot(counter, loss_history) # save variables so they can be plotted later """ with open('objs.pkl', 'wb') as f: pickle.dump([counter, loss_history], f) """ print("Training Completed") # Save the model torch.save(net, 'filename.pt') print("Model is saved")
# device device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # data loaders train_dl = DataLoader(MadoriOutlineSiameseDS(train_file), batch_size=batch_size, shuffle=True) val_dl = DataLoader(MadoriOutlineSiameseDS(val_file), batch_size=batch_size, shuffle=True) # model model = SiameseNetwork().to(device) criterion_unet = DiceLoss() criterion_siamese = ContrastiveLoss() optimizer = Adam(model.parameters(), lr=0.0005) # train unet_train_loss_history, unet_val_loss_history = [], [] sia_train_loss_history, sia_val_loss_history = [], [] unet_lowest_epoch_train_loss = unet_lowest_epoch_val_loss = float('inf') sia_lowest_epoch_train_loss = sia_lowest_epoch_val_loss = float('inf') for epoch in tqdm(range(num_epochs)): model.train() unet_epoch_train_loss = sia_epoch_train_loss = 0 for i, batch in enumerate(train_dl): img1, label1, img2, label2, is_diff = batch img1, label1 = img1.to(device), label1.to(device) img2, label2 = img2.to(device), label2.to(device)
# Output demo: # print (type(train_dataset.train_data[5])) # print (train_dataset.train_data[5].size()) # print (train_dataset.train_data[5]) # print (type(train_dataset.train_labels[5])) # print (train_dataset.train_labels[5].size()) # print (train_dataset.train_labels[5]) siamese_train_dataset = SiameseMNIST(train_dataset) siamese_test_dataset = SiameseMNIST(test_dataset) # set up data loaders batch_size = 128 kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} train_loader = torch.utils.data.DataLoader(siamese_train_dataset, batch_size=batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(siamese_test_dataset, batch_size=batch_size, shuffle=False, **kwargs) margin = 1 embedding_net = EmbeddingNet() model = SiameseNet(embedding_net) if cuda: model.cuda() loss_fn = ContrastiveLoss(margin) lr = 1e-3 optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 1 log_interval = 100 fit(train_loader, test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval)
# It should take around 1-2 hours on GPU. # dir_name = os.path.join('./model', name) if not os.path.exists('model'): os.mkdir('model') print('class_num = %d' % (class_num)) embedding_net = ft_net(class_num) model = SiameseNet(embedding_net) if use_gpu: model.cuda() # print('model structure') # print(model) criterion_contrastive = ContrastiveLoss() criterion_verify = nn.CrossEntropyLoss() classifier_id = list(map(id, model.embedding_net.classifier.parameters())) \ + list(map(id, model.classifier.parameters())) classifier_params = filter(lambda p: id(p) in classifier_id, model.parameters()) base_params = filter(lambda p: id(p) not in classifier_id, model.parameters()) optimizer_ft = optim.SGD([ { 'params': classifier_params, 'lr': 1 * opt.lr }, { 'params': base_params,