def test_iris(self): scores = [] for i in range(1): hidden = 50 l1 = Linear(4, hidden, initialize='ones') l2 = Linear(hidden, 3, initialize='ones') l1.W *= 0.000000001 l2.W *= 0.00000001 model = Seq(l1, Sigmoid, l2) loss = CrossEntropyLoss() trainer = OnlineTrainer() losses = trainer.train(model, self.train_set, epochs=100, loss=loss, optimizer=SGD(learning_rate=0.01)) score = loss.test_score(model, self.train_set) print("hidden=%f score=%f" % (hidden, score)) scores.append(score) self.plot_loss_history(losses) plt.show() self.assertGreaterEqual(numpy.mean(scores), 94.)
def test_numerical_gradient(self): x = np.random.rand(5) target_class = make_one_hot_target(classes_n=5, target_class=1) loss = CrossEntropyLoss() y = loss.calc_loss(x, target_class) grad = loss.calc_gradient(y, target_class) def forward(i): return loss.calc_loss(i, target_class) num_grad = numerical_gradient.calc(forward, x) num_grad = np.sum(num_grad, axis=0) print num_grad numerical_gradient.assert_are_similar(grad, num_grad)
def network_setup(model_file_path=None): freq_count = 4000 count_bins = 88 * 20 dataset = MapsDB('../db', freq_count=freq_count, count_bins=count_bins, batch_size=128, start_time=0.5, duration=0.5) model = Network() model.add(Linear('fc1', dataset.get_vec_input_width(), 2048, 0.001)) model.add(Sigmoid('sigmoid1')) model.add(Linear('fc2', 2048, dataset.get_label_width(), 0.001)) model.add(Softmax('softmax2')) loss = CrossEntropyLoss(name='xent') # loss = EuclideanLoss(name='r2') optim = SGDOptimizer(learning_rate=0.00001, weight_decay=0.005, momentum=0.9) # optim = AdagradOptimizer(learning_rate=0.001, eps=1e-6) input_placeholder = T.fmatrix('input') label_placeholder = T.fmatrix('label') label_active_size_placeholder = T.ivector('label_active_size') if model_file_path: model.loads(model_file_path) else: dataset.load_cache() model.compile(input_placeholder, label_placeholder, label_active_size_placeholder, loss, optim) return model, dataset, freq_count, count_bins
def set_model(): model = WideResnet(n_classes, k=wresnet_k, n=wresnet_n) # wide resnet-28 model.train() model.cuda() criteria_x = CrossEntropyLoss().cuda() criteria_u = nn.MSELoss().cuda() return model, criteria_x, criteria_u
def getNetwork(): ''' to obtain network structure from specified file ''' file_name = "models/structure.json" if len(sys.argv)>1: file_name = sys.argv[1] f = file(file_name, "r") s = f.read() f.close() networks = json.loads(s) for network in networks: config = network['config'] dis_model = network['model'] model = Network() for layer in dis_model: if layer['type'] == 'Linear': model.add(Linear(layer['name'], layer['in_num'], layer['out_num'], layer['std'])) if layer['type'] == 'Relu': model.add(Relu(layer['name'])) if layer['type'] == 'Sigmoid': model.add(Sigmoid(layer['name'])) if layer['type'] == 'Softmax': model.add(Softmax(layer['name'])) loss = EuclideanLoss('loss') if 'loss' in config: if config['loss'] == 'CrossEntropyLoss': loss = CrossEntropyLoss('loss') yield network['name'], model, config, loss
def run(self, batch_size=10, learning_rate=0.6, train_set_percentage=1.0, epochs=3): model = Seq( WxBiasLinear(784, 10, initialize='random') # Dropout(0.5) ) train_set_sliced = slice_percentage(self.train_set, train_set_percentage) trainer = MinibatchTrainer() trainer.train_minibatches( model, train_set_sliced, batch_size=batch_size, loss=CrossEntropyLoss(), epochs=epochs, optimizer=SGD(learning_rate=learning_rate), # optimizer=RMSProp(learning_rate=learning_rate, decay_rate=0.6), # optimizer=AdaGrad(learning_rate=learning_rate), show_progress=True) # self.show_mnist_grid(model, self.test_set) # trainer = PatienceTrainer() # trainer.train(model, # train_set_sliced, self.valid_set, self.test_set, # batch_size=batch_size, # loss=CrossEntropyLoss(), # max_epochs=100, # # optimizer=MomentumSGD(learning_rate=learning_rate, momentum=0.5), # optimizer=RMSProp(learning_rate=learning_rate, decay_rate=0.9), # # optimizer=AdaGrad(learning_rate=learning_rate), # test_score_function=self.test_score_fun # ) test_score = CrossEntropyLoss().test_score(model, self.test_set) return { # 'train_score': train_score, 'test_score': test_score, }
def __init__(self, args): super(MultiTaskNet, self).__init__() self.encoder = DGCNN_Seg_Encoder(args) self.decoder = FoldNet_Decoder(args) if args.rec_loss == 'ChamferLoss': self.rec_loss = ChamferLoss() elif args.rec_loss == 'ChamferLoss_m': self.rec_loss = ChamferLoss_m() self.classifer = DGCNN_RT_Cls_Classifier(args) self.rot_loss = CrossEntropyLoss()
def __init__(self, args, seg_num_all=50): super(SegmentationNet, self).__init__() self.is_eval = args.eval if args.encoder == 'foldnet': self.encoder = FoldNet_Encoder(args) elif args.encoder == 'dgcnn_cls': self.encoder = DGCNN_Cls_Encoder(args) elif args.encoder == 'dgcnn_seg': self.encoder = DGCNN_Seg_Encoder(args) if not self.is_eval: self.segmenter = DGCNN_Seg_Segmenter(args, seg_num_all) self.loss = CrossEntropyLoss()
def __init__(self, args): super(ClassificationNet, self).__init__() self.is_eval = args.eval if args.encoder == 'foldnet': self.encoder = FoldNet_Encoder(args) elif args.encoder == 'dgcnn_cls': self.encoder = DGCNN_Cls_Encoder(args) elif args.encoder == 'dgcnn_seg': self.encoder = DGCNN_Seg_Encoder(args) if not self.is_eval: self.classifier = DGCNN_Cls_Classifier(args) self.loss = CrossEntropyLoss()
def run(self): train_set, valid_set, test_set = gen_data(dataset='circles') l1 = 0. l2 = 0. # model = Seq([ # Linear(2, 100), # # Dropout(0.8), # Tanh(), # Linear(100, 4), # # Tanh(), # # Dropout(0.6) # ]) model = SyntaxLayer( syntax.Linear(10, 4, input= syntax.Tanh(syntax.Linear(2, 10, input=syntax.Var('x'))))) # trainer = SimpleTrainer() # trainer.train(model, train_set, # loss=CrossEntropyLoss(), # optimizer=SGD(learning_rate=0.1), # # optimizer=MomentumSGD(learning_rate=0.1, momentum=0.8), # # optimizer=AdaGrad(learning_rate=0.9), # # optimizer=RMSProp(learning_rate=0.1, decay_rate=0.9), # epochs=100) trainer = MinibatchTrainer() batch_size = 40 batches_n = (float(len(train_set)) / batch_size) learning_rate = 0.1 / batches_n # print learning_rate mean_losses = trainer.train_minibatches(model, train_set, batch_size, epochs=100, loss=CrossEntropyLoss(), optimizer=MomentumSGD(learning_rate, momentum=0.4), show_progress=True) # trainer = PatienceTrainer() # mean_losses = trainer.train(model, train_set, valid_set, test_set, # batch_size=1, # max_epochs=10000, # loss=CrossEntropyLoss(), # test_score_function=CrossEntropyLoss.test_score, # optimizer=SGD(learning_rate=0.1)) draw_decision_surface(model) scatter_train_data(train_set) plot_mean_loss(mean_losses) plt.show()
def __init__(self, args, seg_num_all=50): super(SegmentationNet, self).__init__() self.is_eval = args.eval self.loss_type = args.loss if args.encoder == 'foldnet': self.encoder = FoldNet_Encoder(args) elif args.encoder == 'dgcnn_cls': self.encoder = DGCNN_Cls_Encoder(args) elif args.encoder == 'dgcnn_seg': self.encoder = DGCNN_Seg_Encoder(args) if not self.is_eval: self.segmenter = DGCNN_Seg_Segmenter(args, seg_num_all) if self.loss_type == 'softmax': self.loss = CrossEntropyLoss() elif self.loss_type == 'triplet': self.loss = TripletLoss(margin=args.margin)
def test_ClaudioMaxNLL(self): x = np.random.rand(10) y = np.random.rand(10) target_class = 3 loss = CrossEntropyLoss() def f1(): J = loss.calc_loss(x, target_class) dJdy = loss.calc_gradient(y, target_class) cla_loss = ClaudioMaxNLL() def f2(): J = cla_loss.calc_loss(x, target_class) dJdy = cla_loss.calc_gradient(y, target_class) t1 = timeit.timeit(f1, number=10000) t2 = timeit.timeit(f1, number=10000) print(t1, t2) self.assertGreater(t2, t1)
def train(seq, dataloader, epochs=10): criterion = CrossEntropyLoss(seq) optimizer = Adam(seq) for epoch in range(epochs): epoch_loss = 0.0 epoch_accuracy = 0.0 n_batch = 0 for batch, labels in dataloader: n_batch += 1 outputs = seq(batch) loss = criterion(outputs, labels) accuracy = accuracy_score(outputs.argmax(axis=1), labels) loss.backward() optimizer.step() epoch_loss += loss epoch_accuracy += accuracy print("Epoch {}/{} - loss: {:%.5f} accuracy: {:%.5f}".format( epoch + 1, epochs, epoch_loss / n_batch, epoch_accuracy / n_batch)) print("Finished training !")
def main(): #GENERAL torch.cuda.empty_cache() root = "/home/kuru/Desktop/veri-gms-master/" train_dir = '/home/kuru/Desktop/veri-gms-master/VeRispan/image_train/' source = {'verispan'} target = {'verispan'} workers = 4 height = 320 width = 320 train_sampler = 'RandomSampler' #AUGMENTATION random_erase = True jitter = True aug = True #OPTIMIZATION opt = 'adam' lr = 0.001 weight_decay = 5e-4 momentum = 0.9 sgd_damp = 0.0 nesterov = True warmup_factor = 0.01 warmup_method = 'linear' STEPS = (30, 60) GAMMA = 0.1 WARMUP_FACTOR = 0.01 WARMUP_EPOCHS = 10 WARMUP_METHOD = 'linear' #HYPERPARAMETER max_epoch = 80 start = 0 train_batch_size = 16 test_batch_size = 50 #SCHEDULER lr_scheduler = 'multi_step' stepsize = [30, 60] gamma = 0.1 #LOSS margin = 0.3 num_instances = 4 lambda_tri = 1 #MODEL #arch = 'resnet101' arch = 'resnet50_ibn_a' no_pretrained = False #TEST SETTINGS #load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet101_ibn_a.pth' #load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet101_ibn_a.pth' load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet50_ibn_a.pth' #load_weights = None start_eval = 0 eval_freq = -1 num_classes = 776 feat_dim = 2048 CENTER_LR = 0.5 CENTER_LOSS_WEIGHT = 0.0005 center_criterion = CenterLoss(num_classes=num_classes, feat_dim=feat_dim, use_gpu=True) optimizer_center = torch.optim.SGD(center_criterion.parameters(), lr=CENTER_LR) #MISC use_gpu = True #use_gpu = False print_freq = 10 seed = 1 resume = '' save_dir = '/home/kuru/Desktop/veri-gms-master_noise/spanningtree_veri_pure/' gpu_id = 0, 1 vis_rank = True query_remove = True evaluate = False dataset_kwargs = { 'source_names': source, 'target_names': target, 'root': root, 'height': height, 'width': width, 'train_batch_size': train_batch_size, 'test_batch_size': test_batch_size, 'train_sampler': train_sampler, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } transform_kwargs = { 'height': height, 'width': width, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } optimizer_kwargs = { 'optim': opt, 'lr': lr, 'weight_decay': weight_decay, 'momentum': momentum, 'sgd_dampening': sgd_damp, 'sgd_nesterov': nesterov } lr_scheduler_kwargs = { 'lr_scheduler': lr_scheduler, 'stepsize': stepsize, 'gamma': gamma } use_gpu = torch.cuda.is_available() log_name = 'log_test.txt' if evaluate else 'log_train.txt' sys.stdout = Logger(osp.join(save_dir, log_name)) print('Currently using GPU ', gpu_id) cudnn.benchmark = True print('Initializing image data manager') #dataset = init_imgreid_dataset(root='/home/kuru/Desktop/veri-gms-master/', name='veri') dataset = init_imgreid_dataset(root='/home/kuru/Desktop/veri-gms-master/', name='verispan') train = [] num_train_pids = 0 num_train_cams = 0 print(len(dataset.train)) for img_path, pid, camid, subid, countid in dataset.train: #print(img_path) path = img_path[56:90 + 6] #print(path) folder = path[1:4] #print(folder) #print(img_path, pid, camid,subid,countid) pid += num_train_pids camid += num_train_cams newidd = 0 train.append((path, folder, pid, camid, subid, countid)) #print(train) #break num_train_pids += dataset.num_train_pids num_train_cams += dataset.num_train_cams pid = 0 pidx = {} for img_path, pid, camid, subid, countid in dataset.train: path = img_path[56:90 + 6] folder = path[1:4] pidx[folder] = pid pid += 1 #print(pidx) sub = [] final = 0 xx = dataset.train newids = [] print(train[0:2]) train2 = {} for k in range(0, 770): for img_path, pid, camid, subid, countid in dataset.train: if k == pid: newid = final + subid sub.append(newid) #print(pid,subid,newid) newids.append(newid) train2[img_path] = newid #print(img_path, pid, camid, subid, countid, newid) final = max(sub) #print(final) print(len(newids), final) #train=train2 #print(train2) train3 = [] for img_path, pid, camid, subid, countid in dataset.train: #print(img_path,pid,train2[img_path]) path = img_path[56:90 + 6] #print(path) folder = path[1:4] newid = train2[img_path] #print((path, folder, pid, camid, subid, countid,newid )) train3.append((path, folder, pid, camid, subid, countid, newid)) train = train3 # for (path, folder, pid, camid, subid, countid,newid) in train: # print(path, folder) #path = '/home/kuru/Desktop/adhi/veri-final-draft-master_noise/gmsNoise776/' path = '/home/kuru/Desktop/veri-gms-master/gms/' pkl = {} #pkl[0] = pickle.load('/home/kuru/Desktop/veri-gms-master/gms/620.pkl') entries = os.listdir(path) for name in entries: f = open((path + name), 'rb') ccc = (path + name) #print(ccc) if name == 'featureMatrix.pkl': s = name[0:13] else: s = name[0:3] #print(s) #with open (ccc,"rb") as ff: # pkl[s] = pickle.load(ff) #print(pkl[s]) pkl[s] = pickle.load(f) f.close #print(len(pkl)) print('=> pickle indexing') data_index = search(pkl) print(len(data_index)) transform_t = train_transforms(**transform_kwargs) #print(train[0],train[10]) #data_tfr = vd(pkl_file='index.pkl', dataset = train, root_dir='/home/kuru/Desktop/veri-gms-master/VeRi/image_train/', transform=transform_t) data_tfr = vdspan( pkl_file='index_veryspan.pkl', dataset=train, root_dir='/home/kuru/Desktop/veri-gms-master/VeRispan/image_train/', transform=transform_t) #print(data_tfr) #print(trainloader) #data_tfr2=list(data_tfr) print("lllllllllllllllllllllllllllllllllllllllllllline 433") df2 = [] data_tfr_old = data_tfr for (img, label, index, pid, cid, subid, countid, newid) in data_tfr: #print((img,label,index,pid, cid,subid,countid,newid) ) #print("datframe",(label)) #print(countid) if countid > 4: #print(countid) df2.append((img, label, index, pid, cid, subid, countid, newid)) print("filtered final trainset length", len(df2)) data_tfr = df2 # with open('df2noise_ex.pkl', 'wb') as handle: # b = pickle.dump(df2, handle, protocol=pickle.HIGHEST_PROTOCOL) # with open('df2noise.pkl', 'rb') as handle: # df2 = pickle.load(handle) # data_tfr=df2 # for (img,label,index,pid, cid,subid,countid,newid) in data_tfr : # print("datframe",(label)) #data_tfr = vdspansort( dataset = train, root_dir='/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/', transform=transform_t) #trainloader = DataLoader(df2, sampler=None,batch_size=train_batch_size, shuffle=True, num_workers=workers,pin_memory=True, drop_last=True) trainloader = DataLoader(data_tfr, sampler=None, batch_size=train_batch_size, shuffle=True, num_workers=workers, pin_memory=True, drop_last=True) for batch_idx, (img, label, index, pid, cid, subid, countid, newid) in enumerate(trainloader): #print("trainloader",batch_idx, (label,index,pid, cid,subid,countid,newid)) print("trainloader", batch_idx, (label)) break print('Initializing test data manager') dm = ImageDataManager(use_gpu, **dataset_kwargs) testloader_dict = dm.return_dataloaders() print('Initializing model: {}'.format(arch)) model = models.init_model(name=arch, num_classes=num_train_pids, loss={'xent', 'htri'}, pretrained=not no_pretrained, last_stride=2) print('Model size: {:.3f} M'.format(count_num_param(model))) if load_weights is not None: print("weights loaded") load_pretrained_weights(model, load_weights) #checkpoint = torch.load('/home/kuru/Desktop/veri-gms-master/logg/model.pth.tar-19') #model._load_from_state_dict(checkpoint['state_dict']) #model.load_state_dict(checkpoint['state_dict']) #optimizer.load_state_dict(checkpoint['optimizer']) #print(checkpoint['epoch']) #print(checkpoint['rank1']) os.environ['CUDA_VISIBLE_DEVICES'] = '0' print(torch.cuda.device_count()) model = nn.DataParallel(model).cuda() if use_gpu else model optimizer = init_optimizer(model, **optimizer_kwargs) #optimizer = init_optimizer(model) #optimizer.load_state_dict(checkpoint['optimizer']) scheduler = init_lr_scheduler(optimizer, **lr_scheduler_kwargs) # scheduler = WarmupMultiStepLR(optimizer, STEPS, GAMMA, # WARMUP_FACTOR, # WARMUP_EPOCHS, WARMUP_METHOD) criterion_xent = CrossEntropyLoss(num_classes=num_train_pids, use_gpu=use_gpu, label_smooth=True) criterion_htri = TripletLoss(margin=margin) ranking_loss = nn.MarginRankingLoss(margin=margin) if evaluate: print('Evaluate only') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] _, distmat = test(model, queryloader, galleryloader, train_batch_size, use_gpu, return_distmat=True) if vis_rank: visualize_ranked_results(distmat, dm.return_testdataset_by_name(name), save_dir=osp.join( save_dir, 'ranked_results', name), topk=20) return time_start = time.time() ranklogger = RankLogger(source, target) # # checkpoint = torch.load('/home/kuru/Desktop/market_all/ibna_model/model.pth.tar-79') # # model.load_state_dict(checkpoint['state_dict']) # # optimizer.load_state_dict(checkpoint['optimizer']) # # print(checkpoint['epoch']) # # start_epoch=checkpoint['epoch'] # # start=start_epoch # checkpoint = torch.load('/home/kuru/Desktop/veri-gms-master/spanningtreeveri/model.pth.tar-2') # model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) # print(checkpoint['epoch']) # start_epoch=checkpoint['epoch'] # start=start_epoch ##start_epoch=resume_from_checkpoint('/home/kuru/Desktop/veri-gms-master/logg/model.pth.tar-20', model, optimizer=None) print('=> Start training') for epoch in range(start, max_epoch): print(epoch, scheduler.get_lr()[0]) #print( torch.cuda.memory_allocated(0)) losses = AverageMeter() #xent_losses = AverageMeter() htri_losses = AverageMeter() accs = AverageMeter() batch_time = AverageMeter() xent_losses = AverageMeter() model.train() for p in model.parameters(): p.requires_grad = True # open all layers end = time.time() for batch_idx, (img, label, index, pid, cid, subid, countid, newid) in enumerate(trainloader): trainX, trainY = torch.zeros( (train_batch_size * 3, 3, height, width), dtype=torch.float32), torch.zeros((train_batch_size * 3), dtype=torch.int64) #pids = torch.zeros((batch_size*3), dtype = torch.int16) #batchcount=0 for i in range(train_batch_size): if (countid[i] > 4): #batchcount=batchcount+1 #print("dfdsfs") labelx = label[i] indexx = index[i] cidx = pid[i] if indexx > len(pkl[labelx]) - 1: indexx = len(pkl[labelx]) - 1 #maxx = np.argmax(pkl[labelx][indexx]) a = pkl[labelx][indexx] minpos = np.argmin(ma.masked_where(a == 0, a)) # print(len(a)) # print(a) # print(ma.masked_where(a==0, a)) # print(labelx,index,pid,cidx,minpos) # print(np.array(data_index).shape) # print(data_index[cidx][1]) pos_dic = data_tfr_old[data_index[cidx][1] + minpos] #print('posdic', pos_dic) neg_label = int(labelx) while True: neg_label = random.choice(range(1, 770)) #print(neg_label) if neg_label is not int(labelx) and os.path.isdir( os.path.join( '/home/kuru/Desktop/veri-gms-master_noise/veriNoise_train_spanning_folder', strint(neg_label))) is True: break negative_label = strint(neg_label) neg_cid = pidx[negative_label] neg_index = random.choice( range(0, len(pkl[negative_label]))) #print(negative_label,neg_cid,neg_index,data_index[neg_cid] ) neg_dic = data_tfr_old[data_index[neg_cid][1] + neg_index] #print('negdic', neg_dic) trainX[i] = img[i] trainX[i + train_batch_size] = pos_dic[0] trainX[i + (train_batch_size * 2)] = neg_dic[0] trainY[i] = cidx trainY[i + train_batch_size] = pos_dic[3] trainY[i + (train_batch_size * 2)] = neg_dic[3] # trainY[i+train_batch_size] = pos_dic[7] # trainY[i+(train_batch_size*2)] = neg_dic[7] #break # else: # print("skiped",countid[i],subid[i],label[i]) #break #print(batchcount) trainX = trainX.cuda() trainY = trainY.cuda() outputs, features = model(trainX) xent_loss = criterion_xent(outputs[0:train_batch_size], trainY[0:train_batch_size]) htri_loss = criterion_htri(features, trainY) centerloss = CENTER_LOSS_WEIGHT * center_criterion( features, trainY) #tri_loss = ranking_loss(features) #ent_loss = xent_loss(outputs[0:batch_size], trainY[0:batch_size], num_train_pids) loss = htri_loss + xent_loss + centerloss loss = htri_loss + xent_loss optimizer.zero_grad() optimizer_center.zero_grad() loss.backward() optimizer.step() # for param in center_criterion.parameters(): # param.grad.data *= (1. /CENTER_LOSS_WEIGHT) # optimizer_center.step() for param_group in optimizer.param_groups: #print(param_group['lr'] ) lrrr = str(param_group['lr']) batch_time.update(time.time() - end) losses.update(loss.item(), trainY.size(0)) htri_losses.update(htri_loss.item(), trainY.size(0)) xent_losses.update(xent_loss.item(), trainY.size(0)) accs.update( accuracy(outputs[0:train_batch_size], trainY[0:train_batch_size])[0]) if (batch_idx) % 50 == 0: print('Train ', end=" ") print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'TriLoss {loss.val:.4f} ({loss.avg:.4f})\t' 'XLoss {xloss.val:.4f} ({xloss.avg:.4f})\t' 'OveralLoss {oloss.val:.4f} ({oloss.avg:.4f})\t' 'Acc {acc.val:.2f} ({acc.avg:.2f})\t' 'lr {lrrr} \t'.format( epoch + 1, batch_idx + 1, len(trainloader), batch_time=batch_time, loss=htri_losses, xloss=xent_losses, oloss=losses, acc=accs, lrrr=lrrr, )) end = time.time() # del loss # del htri_loss # del xent_loss # del htri_losses # del losses # del outputs # del features # del accs # del trainX # del trainY scheduler.step() print('=> Test') save_checkpoint( { 'state_dict': model.state_dict(), #'rank1': rank1, 'epoch': epoch + 1, 'arch': arch, 'optimizer': optimizer.state_dict(), }, save_dir) GPUtil.showUtilization() print(torch.cuda.memory_allocated(), torch.cuda.memory_cached()) for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] rank1, distmat = test(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank1) rank2, distmat2 = test_rerank(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank2) del queryloader del galleryloader del distmat print(torch.cuda.memory_allocated(), torch.cuda.memory_cached()) torch.cuda.empty_cache() if (epoch + 1) == max_epoch: #if (epoch + 1) % 10 == 0: print('=> Test') save_checkpoint( { 'state_dict': model.state_dict(), 'rank1': rank1, 'epoch': epoch + 1, 'arch': arch, 'optimizer': optimizer.state_dict(), }, save_dir) for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] rank1, distmat = test(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank1) # del queryloader # del galleryloader # del distmat if vis_rank: visualize_ranked_results( distmat, dm.return_testdataset_by_name(name), save_dir=osp.join(save_dir, 'ranked_results', name), topk=20)
def main(): #GENERAL torch.cuda.empty_cache() root = "/home/kuru/Desktop/veri-gms-master_noise/" train_dir = '/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/' source = {'verispan'} target = {'verispan'} workers = 4 height = 280 width = 280 train_size = 32 train_sampler = 'RandomSampler' #AUGMENTATION random_erase = True jitter = True aug = True #OPTIMIZATION opt = 'adam' lr = 0.0003 weight_decay = 5e-4 momentum = 0.9 sgd_damp = 0.0 nesterov = True warmup_factor = 0.01 warmup_method = 'linear' #HYPERPARAMETER max_epoch = 80 start = 0 train_batch_size = 8 test_batch_size = 100 #SCHEDULER lr_scheduler = 'multi_step' stepsize = [30, 60] gamma = 0.1 #LOSS margin = 0.3 num_instances = 4 lambda_tri = 1 #MODEL #arch = 'resnet101' arch='resnet101_ibn_a' no_pretrained = False #TEST SETTINGS load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet101_ibn_a.pth' #load_weights = None start_eval = 0 eval_freq = -1 #MISC use_gpu = True print_freq = 10 seed = 1 resume = '' save_dir = '/home/kuru/Desktop/veri-gms-master_noise/spanningtree_verinoise_101_stride2/' gpu_id = 0,1 vis_rank = True query_remove = True evaluate = False dataset_kwargs = { 'source_names': source, 'target_names': target, 'root': root, 'height': height, 'width': width, 'train_batch_size': train_batch_size, 'test_batch_size': test_batch_size, 'train_sampler': train_sampler, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } transform_kwargs = { 'height': height, 'width': width, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } optimizer_kwargs = { 'optim': opt, 'lr': lr, 'weight_decay': weight_decay, 'momentum': momentum, 'sgd_dampening': sgd_damp, 'sgd_nesterov': nesterov } lr_scheduler_kwargs = { 'lr_scheduler': lr_scheduler, 'stepsize': stepsize, 'gamma': gamma } use_gpu = torch.cuda.is_available() log_name = 'log_test.txt' if evaluate else 'log_train.txt' sys.stdout = Logger(osp.join(save_dir, log_name)) print('Currently using GPU ', gpu_id) cudnn.benchmark = True print('Initializing image data manager') dataset = init_imgreid_dataset(root='/home/kuru/Desktop/veri-gms-master_noise/', name='verispan') train = [] num_train_pids = 0 num_train_cams = 0 print(len( dataset.train)) for img_path, pid, camid, subid, countid in dataset.train: #print(img_path) path = img_path[56+6:90+6] #print(path) folder = path[1:4] #print(folder) pid += num_train_pids newidd=0 train.append((path, folder, pid, camid,subid,countid)) num_train_pids += dataset.num_train_pids num_train_cams += dataset.num_train_cams pid = 0 pidx = {} for img_path, pid, camid, subid, countid in dataset.train: path = img_path[56+6:90+6] folder = path[1:4] pidx[folder] = pid pid+= 1 sub=[] final=0 xx=dataset.train newids=[] print(train[0:2]) train2={} for k in range(0,770): for img_path, pid, camid, subid, countid in dataset.train: if k==pid: newid=final+subid sub.append(newid) #print(pid,subid,newid) newids.append(newid) train2[img_path]= newid #print(img_path, pid, camid, subid, countid, newid) final=max(sub) #print(final) print(len(newids),final) #train=train2 #print(train2) train3=[] for img_path, pid, camid, subid, countid in dataset.train: #print(img_path,pid,train2[img_path]) path = img_path[56:90+6] #print(path) folder = path[1:4] newid=train2[img_path] #print((path, folder, pid, camid, subid, countid,newid )) train3.append((path, folder, pid, camid, subid, countid,newid )) train = train3 path = '/home/kuru/Desktop/adhi/veri-final-draft-master_noise/gmsNoise776/' pkl = {} #pkl[0] = pickle.load('/home/kuru/Desktop/veri-gms-master/gms/620.pkl') entries = os.listdir(path) for name in entries: f = open((path+name), 'rb') ccc=(path+name) #print(ccc) if name=='featureMatrix.pkl': s = name[0:13] else: s = name[0:3] #print(s) #with open (ccc,"rb") as ff: # pkl[s] = pickle.load(ff) #print(pkl[s]) pkl[s] = pickle.load(f) f.close #print(len(pkl)) with open('cids.pkl', 'rb') as handle: b = pickle.load(handle) #print(b) with open('index.pkl', 'rb') as handle: c = pickle.load(handle) transform_t = train_transforms(**transform_kwargs) data_tfr = vdspan(pkl_file='index_veryspan_noise.pkl', dataset = train, root_dir='/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/', transform=transform_t) print("lllllllllllllllllllllllllllllllllllllllllllline 433") df2=[] data_tfr_old=data_tfr for (img,label,index,pid, cid,subid,countid,newid) in data_tfr : #print((img,label,index,pid, cid,subid,countid,newid) ) #print("datframe",(label)) #print(countid) if countid > 4 : #print(countid) df2.append((img,label,index,pid, cid,subid,countid,newid)) print("filtered final trainset length",len(df2)) data_tfr=df2 trainloader = DataLoader(data_tfr, sampler=None,batch_size=train_batch_size, shuffle=True, num_workers=workers,pin_memory=True, drop_last=True) #data_tfr = vd(pkl_file='index.pkl', dataset = train, root_dir=train_dir,transform=transforms.Compose([Rescale(64),RandomCrop(32),ToTensor()])) #dataloader = DataLoader(data_tfr, batch_size=batch_size, shuffle=False, num_workers=0) for batch_idx, (img,label,index,pid, cid,subid,countid,newid) in enumerate(trainloader): #print("trainloader",batch_idx, (label,index,pid, cid,subid,countid,newid)) print("trainloader",batch_idx, (label)) break print('Initializing test data manager') dm = ImageDataManager(use_gpu, **dataset_kwargs) testloader_dict = dm.return_dataloaders() print('Initializing model: {}'.format(arch)) model = models.init_model(name=arch, num_classes=num_train_pids, loss={'xent', 'htri'}, pretrained=not no_pretrained, last_stride =2 ) print('Model size: {:.3f} M'.format(count_num_param(model))) if load_weights is not None: print("weights loaded") load_pretrained_weights(model, load_weights) print(torch.cuda.device_count()) model = nn.DataParallel(model).cuda() if use_gpu else model optimizer = init_optimizer(model, **optimizer_kwargs) #optimizer = init_optimizer(model) scheduler = init_lr_scheduler(optimizer, **lr_scheduler_kwargs) criterion_xent = CrossEntropyLoss(num_classes=num_train_pids, use_gpu=use_gpu, label_smooth=True) criterion_htri = TripletLoss(margin=margin) ranking_loss = nn.MarginRankingLoss(margin = margin) if evaluate: print('Evaluate only') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] _, distmat = test(model, queryloader, galleryloader, train_batch_size, use_gpu, return_distmat=True) if vis_rank: visualize_ranked_results( distmat, dm.return_testdataset_by_name(name), save_dir=osp.join(save_dir, 'ranked_results', name), topk=20 ) return time_start = time.time() ranklogger = RankLogger(source, target) print('=> Start training') data_index = search(pkl) print(len(data_index)) for epoch in range(start, max_epoch): losses = AverageMeter() #xent_losses = AverageMeter() htri_losses = AverageMeter() accs = AverageMeter() batch_time = AverageMeter() xent_losses=AverageMeter() model.train() for p in model.parameters(): p.requires_grad = True # open all layers end = time.time() for batch_idx, (img,label,index,pid, cid,subid,countid,newid) in enumerate(trainloader): trainX, trainY = torch.zeros((train_batch_size*3,3,height, width), dtype=torch.float32), torch.zeros((train_batch_size*3), dtype = torch.int64) #pids = torch.zeros((batch_size*3), dtype = torch.int16) for i in range(train_batch_size): #print("dfdsfs") labelx = label[i] indexx = index[i] cidx = pid[i] if indexx >len(pkl[labelx])-1: indexx = len(pkl[labelx])-1 #maxx = np.argmax(pkl[labelx][indexx]) a = pkl[labelx][indexx] minpos = np.argmin(ma.masked_where(a==0, a)) #print(minpos) #print(np.array(data_index).shape) #print(data_index[cidx][1]) pos_dic = data_tfr_old[data_index[cidx][1]+minpos] neg_label = int(labelx) while True: neg_label = random.choice(range(1, 770)) #print(neg_label) if neg_label is not int(labelx) and os.path.isdir(os.path.join('/home/kuru/Desktop/adiusb/veri-split/train', strint(neg_label))) is True: break negative_label = strint(neg_label) neg_cid = pidx[negative_label] neg_index = random.choice(range(0, len(pkl[negative_label]))) neg_dic = data_tfr_old[data_index[neg_cid][1]+neg_index] trainX[i] = img[i] trainX[i+train_batch_size] = pos_dic[0] trainX[i+(train_batch_size*2)] = neg_dic[0] trainY[i] = cidx trainY[i+train_batch_size] = pos_dic[3] trainY[i+(train_batch_size*2)] = neg_dic[3] trainX = trainX.cuda() trainY = trainY.cuda() outputs, features = model(trainX) xent_loss = criterion_xent(outputs[0:train_batch_size], trainY[0:train_batch_size]) htri_loss = criterion_htri(features, trainY) #tri_loss = ranking_loss(features) #ent_loss = xent_loss(outputs[0:batch_size], trainY[0:batch_size], num_train_pids) loss = htri_loss+xent_loss optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) losses.update(loss.item(), trainY.size(0)) htri_losses.update(htri_loss.item(), trainY.size(0)) xent_losses.update(xent_loss.item(), trainY.size(0)) accs.update(accuracy(outputs[0:train_batch_size], trainY[0:train_batch_size])[0]) if (batch_idx) % 50 == 0: print('Train ', end=" ") print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'TriLoss {loss.val:.4f} ({loss.avg:.4f})\t' 'XLoss {xloss.val:.4f} ({xloss.avg:.4f})\t' 'OveralLoss {oloss.val:.4f} ({oloss.avg:.4f})\t' 'Acc {acc.val:.2f} ({acc.avg:.2f})\t' 'lr {lrrr} \t'.format( epoch + 1, batch_idx + 1, len(trainloader), batch_time=batch_time, loss = htri_losses, xloss = xent_losses, oloss = losses, acc=accs , lrrr=lrrr, )) end = time.time() scheduler.step() print('=> Test') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] rank1, distmat = test(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank1) rank2, distmat2 = test_rerank(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank2) #if (epoch + 1) == max_epoch: if (epoch + 1) % 2 == 0: print('=> Test') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] rank1, distmat = test(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank1) # if vis_rank: # visualize_ranked_results( # distmat, dm.return_testdataset_by_name(name), # save_dir=osp.join(save_dir, 'ranked_results', name), # topk=20) save_checkpoint({ 'state_dict': model.state_dict(), 'rank1': rank1, 'epoch': epoch + 1, 'arch': arch, 'optimizer': optimizer.state_dict(), }, save_dir)
def train_epoch(self): if self.epoch % self.val_epoch == 0 or self.epoch == 1: self.validate() self.model.train() train_metrics = runningScore(self.n_classes) train_loss_meter = averageMeter() self.optim.zero_grad() for rgb, ir, target in tqdm.tqdm( self.train_loader, total=len(self.train_loader), desc=f'Train epoch={self.epoch}', ncols=80, leave=False): self.iter += 1 assert self.model.training rgb, ir, target = rgb.to(self.device), ir.to(self.device), target.to(self.device) score = self.model(rgb, ir) # score = self.model(rgb) weight = self.train_loader.dataset.class_weight if weight: weight = torch.Tensor(weight).to(self.device) loss = CrossEntropyLoss(score, target, weight=weight, ignore_index=-1, reduction='mean') loss_data = loss.data.item() train_loss_meter.update(loss_data) if np.isnan(loss_data): raise ValueError('loss is nan while training') # loss.backward(retain_graph=True) loss.backward() self.optim.step() self.optim.zero_grad() if isinstance(score, (tuple, list)): lbl_pred = score[0].data.max(1)[1].cpu().numpy() else: lbl_pred = score.data.max(1)[1].cpu().numpy() lbl_true = target.data.cpu().numpy() train_metrics.update(lbl_true, lbl_pred) acc, acc_cls, mean_iou, fwavacc, _ = train_metrics.get_scores() metrics = [acc, acc_cls, mean_iou, fwavacc] with open(osp.join(self.out, 'log.csv'), 'a') as f: elapsed_time = ( datetime.datetime.now(pytz.timezone('UTC')) - self.timestamp_start).total_seconds() log = [self.epoch] + [train_loss_meter.avg] + \ metrics + [''] * 5 + [elapsed_time] log = map(str, log) f.write(','.join(log) + '\n') if self.scheduler: self.scheduler.step() if self.epoch % self.val_epoch == 0 or self.epoch == 1: lr = self.optim.param_groups[0]['lr'] print(f'\nCurrent base learning rate of epoch {self.epoch}: {lr:.7f}') train_loss_meter.reset() train_metrics.reset()
def validate(self): visualizations = [] val_metrics = runningScore(self.n_classes) val_loss_meter = averageMeter() with torch.no_grad(): self.model.eval() for rgb, ir, target in tqdm.tqdm( self.val_loader, total=len(self.val_loader), desc=f'Valid epoch={self.epoch}', ncols=80, leave=False): rgb, ir, target = rgb.to(self.device), ir.to(self.device), target.to(self.device) score = self.model(rgb, ir) # score = self.model(rgb) weight = self.val_loader.dataset.class_weight if weight: weight = torch.Tensor(weight).to(self.device) loss = CrossEntropyLoss(score, target, weight=weight, reduction='mean', ignore_index=-1) loss_data = loss.data.item() if np.isnan(loss_data): raise ValueError('loss is nan while validating') val_loss_meter.update(loss_data) rgbs = rgb.data.cpu() irs = ir.data.cpu() if isinstance(score, (tuple, list)): lbl_pred = score[0].data.max(1)[1].cpu().numpy() else: lbl_pred = score.data.max(1)[1].cpu().numpy() lbl_true = target.data.cpu() for rgb, ir, lt, lp in zip(rgbs, irs, lbl_true, lbl_pred): rgb, ir, lt = self.val_loader.dataset.untransform(rgb, ir, lt) val_metrics.update(lt, lp) if len(visualizations) < 9: viz = visualize_segmentation( lbl_pred=lp, lbl_true=lt, img=rgb, ir=ir, n_classes=self.n_classes, dataloader=self.train_loader) visualizations.append(viz) acc, acc_cls, mean_iou, fwavacc, cls_iu = val_metrics.get_scores() metrics = [acc, acc_cls, mean_iou, fwavacc] print(f'\nEpoch: {self.epoch}', f'loss: {val_loss_meter.avg}, mIoU: {mean_iou}') out = osp.join(self.out, 'visualization_viz') if not osp.exists(out): os.makedirs(out) out_file = osp.join(out, 'epoch{:0>5d}.jpg'.format(self.epoch)) scipy.misc.imsave(out_file, get_tile_image(visualizations)) with open(osp.join(self.out, 'log.csv'), 'a') as f: elapsed_time = ( datetime.datetime.now(pytz.timezone('UTC')) - self.timestamp_start).total_seconds() log = [self.epoch] + [''] * 5 + \ [val_loss_meter.avg] + metrics + [elapsed_time] log = map(str, log) f.write(','.join(log) + '\n') mean_iu = metrics[2] is_best = mean_iu > self.best_mean_iu if is_best: self.best_mean_iu = mean_iu torch.save({ 'epoch': self.epoch, 'arch': self.model.__class__.__name__, 'optim_state_dict': self.optim.state_dict(), 'model_state_dict': self.model.state_dict(), 'best_mean_iu': self.best_mean_iu, }, osp.join(self.out, 'checkpoint.pth.tar')) if is_best: shutil.copy(osp.join(self.out, 'checkpoint.pth.tar'), osp.join(self.out, 'model_best.pth.tar')) val_loss_meter.reset() val_metrics.reset() class_name = self.val_loader.dataset.class_names if class_name is not None: for index, value in enumerate(cls_iu.values()): offset = 20 - len(class_name[index]) print(class_name[index] + ' ' * offset + f'{value * 100:>.2f}') else: print("\nyou don't specify class_names, use number instead") for key, value in cls_iu.items(): print(key, f'{value * 100:>.2f}')
def main(): args = parse_args() update_config(cfg, args) # cudnn related setting cudnn.benchmark = cfg.CUDNN.BENCHMARK torch.backends.cudnn.deterministic = cfg.CUDNN.DETERMINISTIC torch.backends.cudnn.enabled = cfg.CUDNN.ENABLED # Set the random seed manually for reproducibility. np.random.seed(cfg.SEED) torch.manual_seed(cfg.SEED) torch.cuda.manual_seed_all(cfg.SEED) # Loss criterion = CrossEntropyLoss(cfg.MODEL.NUM_CLASSES).cuda() # model and optimizer print(f"Definining network with {cfg.MODEL.LAYERS} layers...") model = Network(cfg.MODEL.INIT_CHANNELS, cfg.MODEL.NUM_CLASSES, cfg.MODEL.LAYERS, criterion, primitives_2, drop_path_prob=cfg.TRAIN.DROPPATH_PROB) model = model.cuda() # weight params arch_params = list(map(id, model.arch_parameters())) weight_params = filter(lambda p: id(p) not in arch_params, model.parameters()) # Optimizer optimizer = optim.Adam( weight_params, lr=cfg.TRAIN.LR ) # resume && make log dir and logger if args.load_path and os.path.exists(args.load_path): checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint_best.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) # load checkpoint begin_epoch = checkpoint['epoch'] last_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) best_acc1 = checkpoint['best_acc1'] optimizer.load_state_dict(checkpoint['optimizer']) args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info("=> loaded checkpoint '{}'".format(checkpoint_file)) else: exp_name = args.cfg.split('/')[-1].split('.')[0] args.path_helper = set_path('logs_search', exp_name) logger = create_logger(args.path_helper['log_path']) begin_epoch = cfg.TRAIN.BEGIN_EPOCH best_acc1 = 0.0 last_epoch = -1 logger.info(args) logger.info(cfg) # copy model file this_dir = os.path.dirname(__file__) shutil.copy2( os.path.join(this_dir, 'models', cfg.MODEL.NAME + '.py'), args.path_helper['ckpt_path']) # Datasets and dataloaders # The toy dataset is downloaded with 10 items for each partition. Remove the sample_size parameters to use the full toy dataset asv_train, asv_dev, asv_eval = asv_toys(sample_size=10) train_dataset = asv_train #MNIST('mydata', transform=totensor, train=True, download=True) val_dataset = asv_dev #MNIST('mydata', transform=totensor, train=False, download=True) train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) print(f'search.py: Train loader of {len(train_loader)} batches') print(f'Tot train set: {len(train_dataset)}') val_loader = torch.utils.data.DataLoader( dataset=val_dataset, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) print(f'search.py: Val loader of {len(val_loader)} batches') print(f'Tot val set {len(val_dataset)}') test_dataset = asv_eval #MNIST('mydata', transform=totensor, train=False, download=True) test_loader = torch.utils.data.DataLoader( dataset=test_dataset, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) # training setting writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': begin_epoch * len(train_loader), 'valid_global_steps': begin_epoch // cfg.VAL_FREQ, } # training loop architect = Architect(model, cfg) lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, cfg.TRAIN.END_EPOCH, cfg.TRAIN.LR_MIN, last_epoch=last_epoch ) for epoch in tqdm(range(begin_epoch, cfg.TRAIN.END_EPOCH), desc='search progress'): model.train() genotype = model.genotype() logger.info('genotype = %s', genotype) if cfg.TRAIN.DROPPATH_PROB != 0: model.drop_path_prob = cfg.TRAIN.DROPPATH_PROB * epoch / (cfg.TRAIN.END_EPOCH - 1) train(cfg, model, optimizer, train_loader, val_loader, criterion, architect, epoch, writer_dict) if epoch % cfg.VAL_FREQ == 0: # get threshold and evaluate on validation set acc = validate_identification(cfg, model, test_loader, criterion) # remember best acc@1 and save checkpoint is_best = acc > best_acc1 best_acc1 = max(acc, best_acc1) # save logger.info('=> saving checkpoint to {}'.format(args.path_helper['ckpt_path'])) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), 'arch': model.arch_parameters(), 'genotype': genotype, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'], 'checkpoint_{}.pth'.format(epoch)) lr_scheduler.step(epoch)
perEpooch = len(trainLoader) maxIter = maxEpoch * perEpooch print('Dataset statistics') print("data['classWeights']: ", datas['classWeights']) print('mean and std: ', datas['mean'], datas['std']) # define loss function, respectively weight = torch.from_numpy(datas['classWeights']) # if(lossFunc == 'ohem'): # min_kept = int(batch_size // len(gpus) * h * w // 16) # criteria = ProbOhemCrossEntropy2d(use_weight=True, ignore_label=ignore_label, thresh=0.7, min_kept=min_kept) # elif(lossFunc == 'label_smoothing'): # criteria = CrossEntropyLoss2dLabelSmooth(weight=weight, ignore_label=ignore_label) if (lossFunc == 'CrossEntropy'): criteria = CrossEntropyLoss(weight=weight, ignore_index=ignore_label) elif (lossFunc == 'LovaszSoftmax'): criteria = LovaszSoftmax(ignore_index=ignore_label) elif (lossFunc == 'focal'): criteria = FocalLoss2d(weight=weight, ignore_index=ignore_label) else: raise NotImplementedError( 'We only support CrossEntropy, LovaszSoftmax and focal as loss function.' ) if use_cuda: criteria = criteria.to(device) if torch.cuda.device_count() > 1: print("torch.cuda.device_count()=", torch.cuda.device_count()) gpu_nums = torch.cuda.device_count() model = nn.DataParallel(model).to(device) # multi-card data parallel
def main(): # Training settings parser = argparse.ArgumentParser(description='Cifar10 Example') parser.add_argument('--batch-size', type=int, default=128, metavar='N', help='input batch size for training (default: 128)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=25, metavar='N', help='number of epochs to train (default: 25)') parser.add_argument('--lr', type=float, default=0.1, metavar='LR', help='learning rate (default: 0.1)') parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum (default: 0.9)') parser.add_argument('--model-path', type=str, default='', metavar='M', help='model param path') parser.add_argument('--loss-type', type=str, default='CE', metavar='L', help='B or CE or F or ICF_CE or ICF_F or CB_CE or CB_F') parser.add_argument('--beta', type=float, default=0.999, metavar='B', help='Beta for ClassBalancedLoss') parser.add_argument('--gamma', type=float, default=2.0, metavar='G', help='Gamma for FocalLoss') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--balanced-data', action='store_true', default=False, help='For sampling rate. Default is Imbalanced-data.') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') args = parser.parse_args() # Add the following code anywhere in your machine learning file experiment = Experiment(api_key="5Yl3Rxz9S3E0PUKQTBpA0QJPi", project_name="imbalanced-cifar-10", workspace="tancoro") # ブラウザの実験ページを開く # experiment.display(clear=True, wait=True, new=0, autoraise=True) # 実験キー(実験を一意に特定するためのキー)の取得 exp_key = experiment.get_key() print('KEY: ' + exp_key) # HyperParamの記録 hyper_params = { 'batch_size': args.batch_size, 'epoch': args.epochs, 'learning_rate': args.lr, 'sgd_momentum' : args.momentum, 'model_path' : args.model_path, 'loss_type' : args.loss_type, 'beta' : args.beta, 'gamma' : args.gamma, 'torch_manual_seed': args.seed, 'balanced_data' : args.balanced_data } experiment.log_parameters(hyper_params) use_cuda = not args.no_cuda and torch.cuda.is_available() print('use_cuda {}'.format(use_cuda)) torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} # train dataset cifar10_train_dataset = datasets.CIFAR10('./data', train=True, download=True, transform=transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) # train sampling rate sampling_rate = {} if not args.balanced_data: sampling_rate = {1:0.05, 4:0.05, 6:0.05} print(sampling_rate) # train Sampler train_sampler = ReductionSampler(cifar10_train_dataset, sampling_rate=sampling_rate) # train loader train_loader = torch.utils.data.DataLoader(cifar10_train_dataset, batch_size=args.batch_size, sampler=train_sampler, **kwargs) # test dataset cifar10_test_dataset = datasets.CIFAR10('./data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) # test majority loader test_majority_sampler = ReductionSampler(cifar10_test_dataset, sampling_rate={1:0, 4:0, 6:0}) test_majority_loader = torch.utils.data.DataLoader(cifar10_test_dataset, batch_size=args.test_batch_size, sampler=test_majority_sampler, **kwargs) # test minority loader test_minority_sampler = ReductionSampler(cifar10_test_dataset, sampling_rate={0:0, 2:0, 3:0, 5:0, 7:0, 8:0, 9:0}) test_minority_loader = torch.utils.data.DataLoader(cifar10_test_dataset, batch_size=args.test_batch_size, sampler=test_minority_sampler, **kwargs) # test alldata loader test_alldata_loader = torch.utils.data.DataLoader(cifar10_test_dataset, batch_size=args.test_batch_size, shuffle=True, **kwargs) model = ResNet18().to(device) # train loss train_loss = BasicCrossEntropyLoss() if args.loss_type == 'CE': train_loss = CrossEntropyLoss(train_sampler.get_data_count_map(), device) elif args.loss_type == 'F': train_loss = FocalLoss(train_sampler.get_data_count_map(), device, gamma=args.gamma) elif args.loss_type == 'ICF_CE': train_loss = InverseClassFrequencyCrossEntropyLoss(train_sampler.get_data_count_map(), device) elif args.loss_type == 'ICF_F': train_loss = InverseClassFrequencyFocalLoss(train_sampler.get_data_count_map(), device, gamma=args.gamma) elif args.loss_type == 'CB_CE': train_loss = ClassBalancedCrossEntropyLoss(train_sampler.get_data_count_map(), device, beta=args.beta) elif args.loss_type == 'CB_F': train_loss = ClassBalancedFocalLoss(train_sampler.get_data_count_map(), device, beta=args.beta, gamma=args.gamma) print('Train Loss Type: {}'.format(type(train_loss))) # load param if len(args.model_path) > 0: model.load_state_dict(torch.load(args.model_path)) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=5e-4) # lr = 0.1 if epoch < 15 # lr = 0.01 if 15 <= epoch < 20 # lr = 0.001 if 20 <= epoch < 25 scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15,20], gamma=0.1) for epoch in range(1, args.epochs + 1): with experiment.train(): experiment.log_current_epoch(epoch) train(args, model, device, train_loader, len(train_sampler), optimizer, epoch, experiment, lossfunc=train_loss) with experiment.test(): test(args, model, device, test_minority_loader, len(test_minority_sampler), epoch, experiment, pref='minority') test(args, model, device, test_majority_loader, len(test_majority_sampler), epoch, experiment, pref='majority') test(args, model, device, test_alldata_loader, len(test_alldata_loader.dataset), epoch, experiment, pref='all') if (args.save_model) and (epoch % 10 == 0): print('saving model to ./model/cifar10_{0}_{1:04d}.pt'.format(exp_key, epoch)) torch.save(model.state_dict(), "./model/cifar10_{0}_{1:04d}.pt".format(exp_key, epoch)) scheduler.step()
def main(): args = parse_args() update_config(cfg, args) # cudnn related setting cudnn.benchmark = cfg.CUDNN.BENCHMARK torch.backends.cudnn.deterministic = cfg.CUDNN.DETERMINISTIC torch.backends.cudnn.enabled = cfg.CUDNN.ENABLED # Set the random seed manually for reproducibility. np.random.seed(cfg.SEED) torch.manual_seed(cfg.SEED) torch.cuda.manual_seed_all(cfg.SEED) # Loss criterion = CrossEntropyLoss(cfg.MODEL.NUM_CLASSES).cuda() # load arch genotype = eval(args.text_arch) model = Network(cfg.MODEL.INIT_CHANNELS, cfg.MODEL.NUM_CLASSES, cfg.MODEL.LAYERS, genotype) model = model.cuda() optimizer = optim.Adam( model.parameters(), lr=cfg.TRAIN.LR, weight_decay=cfg.TRAIN.WD, ) # resume && make log dir and logger if args.load_path and os.path.exists(args.load_path): checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint_best.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) # load checkpoint begin_epoch = checkpoint['epoch'] last_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) best_acc1 = checkpoint['best_acc1'] optimizer.load_state_dict(checkpoint['optimizer']) args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info("=> loaded checkloggpoint '{}'".format(checkpoint_file)) else: exp_name = args.cfg.split('/')[-1].split('.')[0] args.path_helper = set_path('logs_scratch', exp_name) logger = create_logger(args.path_helper['log_path']) begin_epoch = cfg.TRAIN.BEGIN_EPOCH best_acc1 = 0.0 last_epoch = -1 logger.info(args) logger.info(cfg) logger.info(f"selected architecture: {genotype}") logger.info("Number of parameters: {}".format(count_parameters(model))) # copy model file this_dir = os.path.dirname(__file__) shutil.copy2(os.path.join(this_dir, './models', cfg.MODEL.NAME + '.py'), args.path_helper['ckpt_path']) # dataloader train_dataset = DeepSpeakerDataset(Path(cfg.DATASET.DATA_DIR), cfg.DATASET.PARTIAL_N_FRAMES, 'train') train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) test_dataset = DeepSpeakerDataset(Path(cfg.DATASET.DATA_DIR), cfg.DATASET.PARTIAL_N_FRAMES, 'test', is_test=True) test_loader = torch.utils.data.DataLoader( dataset=test_dataset, batch_size=1, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) # training setting writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': begin_epoch * len(train_loader), 'valid_global_steps': begin_epoch // cfg.VAL_FREQ, } # training loop lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, cfg.TRAIN.END_EPOCH, cfg.TRAIN.LR_MIN, last_epoch=last_epoch) for epoch in tqdm(range(begin_epoch, cfg.TRAIN.END_EPOCH), desc='train progress'): model.train() model.drop_path_prob = cfg.MODEL.DROP_PATH_PROB * epoch / cfg.TRAIN.END_EPOCH train_from_scratch(cfg, model, optimizer, train_loader, criterion, epoch, writer_dict) if epoch % cfg.VAL_FREQ == 0 or epoch == cfg.TRAIN.END_EPOCH - 1: acc = validate_identification(cfg, model, test_loader, criterion) # remember best acc@1 and save checkpoint is_best = acc > best_acc1 best_acc1 = max(acc, best_acc1) # save logger.info('=> saving checkpoint to {}'.format( args.path_helper['ckpt_path'])) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), 'path_helper': args.path_helper, 'genotype': genotype, }, is_best, args.path_helper['ckpt_path'], 'checkpoint_{}.pth'.format(epoch)) lr_scheduler.step(epoch)
def mnist_model(): model = network.Network() model.add(FullyConnected(in_feature=784, out_feature=512), name='fc1') model.add(LeakyReLU(), name='leaky_relu1') model.add(FullyConnected(in_feature=512, out_feature=256), name='fc2') model.add(LeakyReLU(), name='leaky_relu2') model.add(FullyConnected(in_feature=256, out_feature=256), name='fc3') model.add(LeakyReLU(), name='leaky_relu3') model.add(FullyConnected(in_feature=256, out_feature=128), name='fc4') model.add(LeakyReLU(), name='leaky_relu4') model.add(FullyConnected(in_feature=128, out_feature=10), name='fc5') model.add(Softmax(), name='softmax') model.add_loss(CrossEntropyLoss()) optimizer = SGD(lr=1e-4) print(model) traingset = fetch_traingset() train_images, train_labels = traingset['images'], traingset['labels'] batch_size = 256 training_size = len(train_images) loss_list = np.zeros((50, int(training_size / batch_size))) for epoch in range(50): for i in range(int(training_size / batch_size)): batch_images = np.array(train_images[i * batch_size:(i + 1) * batch_size]) batch_labels = np.array(train_labels[i * batch_size:(i + 1) * batch_size]) batch_labels = one_hot(batch_labels, 10) _, loss = model.forward(batch_images, batch_labels) if i % 50 == 0: loss_list[epoch][i] = loss print("e:{}, i:{} loss: {}".format(epoch, i, loss)) model.backward() model.optimize(optimizer) filename = 'model.data' f = open(filename, 'wb') pickle.dump(model, f) f.close() loss_fname = 'loss.data' f = open(loss_fname, 'wb') pickle.dump(loss_list, f) f.close() testset = fetch_testingset() test_images, test_labels = testset['images'], testset['labels'] test_images = np.array(test_images[:]) test_labels_one_hot = one_hot(test_labels, 10) y_, test_loss = model.forward(test_images, test_labels_one_hot) test_labels_pred = testResult2labels(y_) test_labels = np.array(test_labels) right_num = np.sum(test_labels == test_labels_pred) accuracy = 1.0 * right_num / test_labels.shape[0] print('test accuracy is: ', accuracy) a = 0
def train(network_name='unet', nb_of_epochs=50): context = None if network_name == 'unet': network = UNet(input_dimensions=np.array([56, 56, 56, 1]), nb_of_classes=2, depth=2) batch_manager = DeepMedicBatchManager(input_tile_dimensions=network.input_dimensions, output_tile_dimensions=network.output_dimensions[:-1], training_directory='D:\untitled\ku\new', nb_of_training_images=24, nb_of_training_images_to_load=18, nb_of_tiles_per_training_image=1000, nb_of_tiles_per_training_batch=10, nb_of_testing_images=6, nb_of_tiles_per_testing_image=1000, nb_of_tiles_per_testing_batch=20, tile_foreground_to_background_ratio=0.2) trainer = Trainer(initial_learning_rate=0.0005, loss_function=CrossEntropyLoss()) context = Context(network=network, batch_manager=batch_manager, trainer=trainer, nb_of_training_tiles_before_evaluation=1800) elif network_name == 'vnet': network = VNet(input_dimensions=np.array([64, 64, 64, 1]), nb_of_classes=2, depth=4) batch_manager = DeepMedicBatchManager(input_tile_dimensions=network.input_dimensions, output_tile_dimensions=network.output_dimensions[:-1], training_directory='D:\untitled\ku\new', nb_of_training_images=24, nb_of_training_images_to_load=18, nb_of_tiles_per_training_image=1000, nb_of_tiles_per_training_batch=10, nb_of_testing_images=6, nb_of_tiles_per_testing_image=1000, nb_of_tiles_per_testing_batch=20, tile_foreground_to_background_ratio=0.2) trainer = Trainer(initial_learning_rate=0.001, loss_function=CrossEntropyLoss()) context = Context(network=network, batch_manager=batch_manager, trainer=trainer, nb_of_training_tiles_before_evaluation=1800) elif network_name == 'deepmedic': network = DeepMedic(input_dimensions=np.array([57, 57, 57, 1]), sampling_factor=3, nb_of_classes=2) batch_manager = DeepMedicBatchManager(input_tile_dimensions=network.input_dimensions, output_tile_dimensions=network.output_dimensions[:-1], training_directory='D:\untitled\ku\new', nb_of_training_images=24, nb_of_training_images_to_load=18, nb_of_tiles_per_training_image=1000, nb_of_tiles_per_training_batch=200, # 20 works well, let's see for 40 nb_of_testing_images=6, nb_of_tiles_per_testing_image=1000, nb_of_tiles_per_testing_batch=400, tile_foreground_to_background_ratio=0.2) trainer = Trainer(initial_learning_rate=0.0005, # 0.00001 works well, but bn allows for much higher learning_rates, so to be checked... loss_function=CrossEntropyLoss()) context = Context(network=network, batch_manager=batch_manager, trainer=trainer, nb_of_training_tiles_before_evaluation=1800) context.build_graph() context.train(nb_of_epochs)
def Train(): ### SEED if args.seed is not None: SEED = args.seed torch.manual_seed(SEED) torch.cuda.manual_seed(SEED) np.random.seed(SEED) torch.backends.cudnn.deterministic = True ##### dataset mean = [0.4914, 0.4822, 0.4465] std = [0.2023, 0.1994, 0.2010] normalize = transforms.Normalize(mean, std) transform_train = transforms.Compose([ # TODO: uncomment these lines for task 3 Padding(padding=4), RandomCrop(size=32), RandomFlip(), transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([ transforms.ToTensor(), normalize, ]) data = getattr(dataset, args.dataset)(batch_size=args.batch_size, transform_train=transform_train, transform_test=transform_test) ##### # model=alexnet(num_classes=args.num_classes) # Uncomment the following line for task 5 model = ResNet() model = nn.DataParallel(model) cudnn.benchmark = True model = model.cuda() if args.calculate_flops: print_model_parm_flops(model) return class_count = [0 for i in range(10)] for i, (inputs, target) in enumerate(data.train): for t in target: class_count[t] += 1 dataset_size = sum(class_count) print('Class Information: ', class_count) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) ce = CrossEntropyLoss() # Un-comment the following statement for task 2 # ce_train = CrossEntropyLoss() # Un-comment the following statement for task 4 ce_train = CrossEntropyLoss(cls_count=class_count, dataset_size=dataset_size) ##### lr multi step schedule def lr_schedule_multistep(epoch): if epoch < 5: factor = (epoch + 1) / 5.0 lr = args.lr * (1 / 3.0 * (1 - factor) + factor) elif epoch < 80: lr = args.lr elif epoch < 90: lr = args.lr * 0.1 else: lr = args.lr * 0.001 for param_group in optimizer.param_groups: param_group['lr'] = lr return lr # finetune if args.resume: checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint['state_dict'], strict=True) start_epoch = checkpoint['epoch'] else: start_epoch = 0 #####Train end_epoch = args.epochs best_acc = 0.0 print(model) for epoch in range(start_epoch, end_epoch): #####adjust learning rate every epoch begining lr = lr_schedule_multistep(epoch) ##### model.train() total = 0.0 correct = 0.0 total_loss = 0.0 for i, (inputs, target) in enumerate(data.train): input, target = inputs.cuda(), target.cuda() logit = model(input) loss = ce_train(logit, target, epo=epoch) total_loss += loss optimizer.zero_grad() loss.backward() optimizer.step() _, predicted = logit.max(1) total += target.size(0) correct += predicted.eq(target).sum().item() acc_n = logit.max(dim=1)[1].eq(target).sum().item() log(args, 'Train acc_n %.5f lr %.5f' % (acc_n / input.size(0), lr)) log(args, 'loss = %.5f' % (total_loss / total)) model.eval() total = 0.0 class_num = torch.zeros(args.num_classes).cuda() correct = torch.zeros(args.num_classes).cuda() for i, (inputs, target) in enumerate(data.test): input, target = inputs.cuda(), target.cuda() logit = model(input) # loss = ce(logit, target) _, predicted = logit.max(1) total += target.size(0) target_one_hot = F.one_hot(target, args.num_classes) predict_one_hot = F.one_hot(predicted, args.num_classes) class_num = class_num + target_one_hot.sum(dim=0).to(torch.float) correct = correct + (target_one_hot + predict_one_hot == 2).sum(dim=0).to(torch.float) acc = correct.sum() / total acc_classes = correct / class_num head_acc = acc_classes[:3].mean() medium_acc = acc_classes[3:6].mean() tail_acc = acc_classes[6:10].mean() if best_acc < acc: best_acc = acc torch.save({ 'state_dict': model.state_dict(), 'epoch': epoch }, "model_best.pth") log( args, "Test epoch=%d acc=%.5f best_acc=%.5f" % (epoch, correct.sum() / total, best_acc)) log(args, "Test " + str(correct / class_num)) log( args, "Test head acc:" + str(head_acc) + " medium acc " + str(medium_acc) + " tail acc " + str(tail_acc) + '\n')
train_data, test_data, train_label, test_label = load_mnist_for_cnn('data') model = Network() model.add(Convolution('conv1', 5, 1, 8, 0.1, 28, batch_size)) # output size: N x 4 x 24 x 24 model.add(Relu('relu1')) model.add(Pooling('pool1', 2)) # output size: N x 4 x 12 x 12 model.add(Convolution('conv2', 5, 8, 16, 0.1, 12, batch_size)) # output size: N x 8 x 10 x 10 model.add(Relu('relu2')) model.add(Pooling('pool2', 2)) # output size: N x 8 x 5 x 5 model.add(Flatten('View')) # my own layer, to reshape the output model.add(Linear('fc3', 16 * 4 * 4, 10, 0.1)) # input reshaped to N x 200 in Linear layer model.add(Softmax('softmax')) loss = CrossEntropyLoss(name='xent') optim = AdagradOptimizer(learning_rate=0.0001, eps=1e-8) input_placeholder = T.ftensor4('input') label_placeholder = T.fmatrix('label') model.compile(input_placeholder, label_placeholder, loss, optim) print 'ada' solve_net(model, train_data, train_label, test_data, test_label, batch_size=batch_size, max_epoch=50, disp_freq=1000,
images = image.T.reshape(batch_size, 1, 28, 28) images = np.asarray(images) img_viewer_examples(images, labels.tolist()[0], greyscale= True) model = Model() model.add(Dense(784, 90)) model.add(ReLU()) model.add(Dense(90, 45)) model.add(ReLU()) model.add(Dense(45, 10)) model.set_loss(CrossEntropyLoss()) optimizer = Adam(model.parameters(), learning_rate = 0.01) lr_schedular = StepLR(optimizer, step_size = 1, gamma=0.1) # weights path path = "./checkpoints/Linear_MINST_weights.sav" # model = load_weights(path) epochs = 6 for epoch in range(epochs): i = 0 for image, label in dataloader: if epoch == 5: model.graph() image = image/255
from data_preparation import load_data from solve_rnn import solve_rnn import theano.tensor as T X_train, y_train, X_test, y_test = load_data() HIDDEN_DIM = 32 INPUT_DIM = 20 OUTPUT_DIM = 10 model = Network() model.add(RNN('rnn1', HIDDEN_DIM, INPUT_DIM, 0.1)) # output shape: 4 x HIDDEN_DIM model.add(Linear('fc', HIDDEN_DIM, OUTPUT_DIM, 0.1)) # output shape: 4 x OUTPUT_DIM model.add(Softmax('softmax')) loss = CrossEntropyLoss('xent') optim = SGDOptimizer(0.01, 0.0001, 0.9) input_placeholder = T.fmatrix('input') label_placeholder = T.fmatrix('label') model.compile(input_placeholder, label_placeholder, loss, optim) MAX_EPOCH = 6 DISP_FREQ = 1000 TEST_FREQ = 10000 solve_rnn(model, X_train, y_train, X_test, y_test, MAX_EPOCH, DISP_FREQ, TEST_FREQ)
def main(): args = parse_args() update_config(cfg, args) # cudnn related setting cudnn.benchmark = cfg.CUDNN.BENCHMARK torch.backends.cudnn.deterministic = cfg.CUDNN.DETERMINISTIC torch.backends.cudnn.enabled = cfg.CUDNN.ENABLED # Set the random seed manually for reproducibility. np.random.seed(cfg.SEED) torch.manual_seed(cfg.SEED) torch.cuda.manual_seed_all(cfg.SEED) # model and optimizer model = eval('resnet.{}(num_classes={})'.format(cfg.MODEL.NAME, cfg.MODEL.NUM_CLASSES)) model = model.cuda() optimizer = optim.Adam( model.net_parameters() if hasattr(model, 'net_parameters') else model.parameters(), lr=cfg.TRAIN.LR, ) # Loss criterion = CrossEntropyLoss(cfg.MODEL.NUM_CLASSES).cuda() # resume && make log dir and logger if args.load_path and os.path.exists(args.load_path): checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint_best.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) # load checkpoint begin_epoch = checkpoint['epoch'] last_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) best_eer = checkpoint['best_eer'] optimizer.load_state_dict(checkpoint['optimizer']) args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info("=> loaded checkpoint '{}'".format(checkpoint_file)) else: exp_name = args.cfg.split('/')[-1].split('.')[0] args.path_helper = set_path('logs', exp_name) logger = create_logger(args.path_helper['log_path']) begin_epoch = cfg.TRAIN.BEGIN_EPOCH best_eer = 1.0 last_epoch = -1 logger.info(args) logger.info(cfg) logger.info("Number of parameters: {}".format(count_parameters(model))) # dataloader train_dataset = DeepSpeakerDataset(Path(cfg.DATASET.DATA_DIR), cfg.DATASET.SUB_DIR, cfg.DATASET.PARTIAL_N_FRAMES) test_dataset_verification = VoxcelebTestset(Path(cfg.DATASET.DATA_DIR), cfg.DATASET.PARTIAL_N_FRAMES) train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=True, drop_last=True, ) test_loader_verification = torch.utils.data.DataLoader( dataset=test_dataset_verification, batch_size=1, num_workers=cfg.DATASET.NUM_WORKERS, pin_memory=True, shuffle=False, drop_last=False, ) # training setting writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': begin_epoch * len(train_loader), 'valid_global_steps': begin_epoch // cfg.VAL_FREQ, } # training loop lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, cfg.TRAIN.END_EPOCH, cfg.TRAIN.LR_MIN, last_epoch=last_epoch) for epoch in tqdm(range(begin_epoch, cfg.TRAIN.END_EPOCH), desc='train progress'): model.train() train_from_scratch(cfg, model, optimizer, train_loader, criterion, epoch, writer_dict, lr_scheduler) if epoch % cfg.VAL_FREQ == 0: eer = validate_verification(cfg, model, test_loader_verification) # remember best acc@1 and save checkpoint is_best = eer < best_eer best_eer = min(eer, best_eer) # save logger.info('=> saving checkpoint to {}'.format( args.path_helper['ckpt_path'])) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_eer': best_eer, 'optimizer': optimizer.state_dict(), 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'], 'checkpoint_{}.pth'.format(epoch)) lr_scheduler.step(epoch)
def main(): #GENERAL root = "/home/kuru/Desktop/veri-gms-master/" train_dir = '/home/kuru/Desktop/veri-gms-master/VeRispan/image_train/' source = {'veri'} target = {'veri'} workers = 2 height = 320 width = 320 train_sampler = 'RandomSampler' #AUGMENTATION random_erase = True jitter = True aug = True #OPTIMIZATION opt = 'adam' lr = 0.0003 weight_decay = 5e-4 momentum = 0.9 sgd_damp = 0.0 nesterov = True warmup_factor = 0.01 warmup_method = 'linear' #HYPERPARAMETER max_epoch = 80 start = 0 train_batch_size = 16 test_batch_size = 50 #SCHEDULER lr_scheduler = 'multi_step' stepsize = [30, 60] gamma = 0.1 #LOSS margin = 0.3 num_instances = 6 lambda_tri = 1 #MODEL arch = 'resnet101_ibn_a' no_pretrained = False #TEST SETTINGS load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet101_ibn_a.pth' #load_weights = None start_eval = 0 eval_freq = -1 #MISC use_gpu = True use_amp = True print_freq = 50 seed = 1 resume = '' save_dir = '/home/kuru/Desktop/veri-gms-master/logapex/' gpu_id = 0 vis_rank = True query_remove = True evaluate = False dataset_kwargs = { 'source_names': source, 'target_names': target, 'root': root, 'height': height, 'width': width, 'train_batch_size': train_batch_size, 'test_batch_size': test_batch_size, 'train_sampler': train_sampler, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } transform_kwargs = { 'height': height, 'width': width, 'random_erase': random_erase, 'color_jitter': jitter, 'color_aug': aug } optimizer_kwargs = { 'optim': opt, 'lr': lr, 'weight_decay': weight_decay, 'momentum': momentum, 'sgd_dampening': sgd_damp, 'sgd_nesterov': nesterov } lr_scheduler_kwargs = { 'lr_scheduler': lr_scheduler, 'stepsize': stepsize, 'gamma': gamma } use_gpu = torch.cuda.is_available() log_name = 'log_test.txt' if evaluate else 'log_train.txt' sys.stdout = Logger(osp.join(save_dir, log_name)) print('Currently using GPU ', gpu_id) cudnn.benchmark = True print('Initializing image data manager') dataset = init_imgreid_dataset(root='/home/kuru/Desktop/veri-gms-master/', name='veri') train = [] num_train_pids = 0 num_train_cams = 0 for img_path, pid, camid in dataset.train: path = img_path[52:77] #print(path) folder = path[1:4] pid += num_train_pids camid += num_train_cams train.append((path, folder, pid, camid)) num_train_pids += dataset.num_train_pids num_train_cams += dataset.num_train_cams pid = 0 pidx = {} for img_path, pid, camid in dataset.train: path = img_path[52:77] folder = path[1:4] pidx[folder] = pid pid += 1 path = '/home/kuru/Desktop/veri-gms-master/gms/' pkl = {} entries = os.listdir(path) for name in entries: f = open((path + name), 'rb') if name == 'featureMatrix.pkl': s = name[0:13] else: s = name[0:3] pkl[s] = pickle.load(f) f.close transform_t = train_transforms(**transform_kwargs) data_tfr = vd( pkl_file='index.pkl', dataset=train, root_dir='/home/kuru/Desktop/veri-gms-master/VeRi/image_train/', transform=transform_t) trainloader = DataLoader(data_tfr, sampler=None, batch_size=train_batch_size, shuffle=True, num_workers=workers, pin_memory=False, drop_last=True) #data_tfr = vd(pkl_file='index.pkl', dataset = train, root_dir=train_dir,transform=transforms.Compose([Rescale(64),RandomCrop(32),ToTensor()])) #dataloader = DataLoader(data_tfr, batch_size=batch_size, shuffle=False, num_workers=0) print('Initializing test data manager') dm = ImageDataManager(use_gpu, **dataset_kwargs) testloader_dict = dm.return_dataloaders() print('Initializing model: {}'.format(arch)) model = models.init_model(name=arch, num_classes=num_train_pids, loss={'xent', 'htri'}, last_stride=1, pretrained=not no_pretrained, use_gpu=use_gpu) print('Model size: {:.3f} M'.format(count_num_param(model))) if load_weights is not None: print("weights loaded") load_pretrained_weights(model, load_weights) model = (model).cuda() if use_gpu else model #model = nn.DataParallel(model).cuda() if use_gpu else model optimizer = init_optimizer(model, **optimizer_kwargs) #optimizer = init_optimizer(model) model, optimizer = amp.initialize(model, optimizer, opt_level="O2", keep_batchnorm_fp32=True, loss_scale="dynamic") model = nn.DataParallel(model).cuda() if use_gpu else model scheduler = init_lr_scheduler(optimizer, **lr_scheduler_kwargs) criterion_xent = CrossEntropyLoss(num_classes=num_train_pids, use_gpu=use_gpu, label_smooth=True) criterion_htri = TripletLoss(margin=margin) ranking_loss = nn.MarginRankingLoss(margin=margin) if evaluate: print('Evaluate only') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] _, distmat = test(model, queryloader, galleryloader, train_batch_size, use_gpu, return_distmat=True) if vis_rank: visualize_ranked_results(distmat, dm.return_testdataset_by_name(name), save_dir=osp.join( save_dir, 'ranked_results', name), topk=20) return time_start = time.time() ranklogger = RankLogger(source, target) print('=> Start training') data_index = search(pkl) for epoch in range(start, max_epoch): losses = AverageMeter() #xent_losses = AverageMeter() htri_losses = AverageMeter() accs = AverageMeter() batch_time = AverageMeter() model.train() for p in model.parameters(): p.requires_grad = True # open all layers end = time.time() for batch_idx, (img, label, index, pid, cid) in enumerate(trainloader): trainX, trainY = torch.zeros( (train_batch_size * 3, 3, height, width), dtype=torch.float32), torch.zeros((train_batch_size * 3), dtype=torch.int64) #pids = torch.zeros((batch_size*3), dtype = torch.int16) for i in range(train_batch_size): labelx = str(label[i]) indexx = int(index[i]) cidx = int(pid[i]) if indexx > len(pkl[labelx]) - 1: indexx = len(pkl[labelx]) - 1 #maxx = np.argmax(pkl[labelx][indexx]) a = pkl[labelx][indexx] minpos = np.argmax(ma.masked_where(a == 0, a)) pos_dic = data_tfr[data_index[cidx][1] + minpos] neg_label = int(labelx) while True: neg_label = random.choice(range(1, 770)) if neg_label is not int(labelx) and os.path.isdir( os.path.join( '/home/kuru/Desktop/adiusb/veri-split/train', strint(neg_label))) is True: break negative_label = strint(neg_label) neg_cid = pidx[negative_label] neg_index = random.choice(range(0, len(pkl[negative_label]))) neg_dic = data_tfr[data_index[neg_cid][1] + neg_index] trainX[i] = img[i] trainX[i + train_batch_size] = pos_dic[0] trainX[i + (train_batch_size * 2)] = neg_dic[0] trainY[i] = cidx trainY[i + train_batch_size] = pos_dic[3] trainY[i + (train_batch_size * 2)] = neg_dic[3] #print("anc",labelx,'posdic', pos_dic[1],pos_dic[2],'negdic', neg_dic[1],neg_dic[2]) trainX = trainX.cuda() trainY = trainY.cuda() outputs, features = model(trainX) xent_loss = criterion_xent(outputs[0:train_batch_size], trainY[0:train_batch_size]) htri_loss = criterion_htri(features, trainY) #tri_loss = ranking_loss(features) #ent_loss = xent_loss(outputs[0:batch_size], trainY[0:batch_size], num_train_pids) loss = htri_loss + xent_loss optimizer.zero_grad() if use_amp: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() #loss.backward() optimizer.step() batch_time.update(time.time() - end) losses.update(loss.item(), trainY.size(0)) htri_losses.update(htri_loss.item(), trainY.size(0)) accs.update( accuracy(outputs[0:train_batch_size], trainY[0:train_batch_size])[0]) if (batch_idx) % print_freq == 0: print('Train ', end=" ") print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.2f} ({acc.avg:.2f})\t'.format( epoch + 1, batch_idx + 1, len(trainloader), batch_time=batch_time, loss=htri_losses, acc=accs)) end = time.time() scheduler.step() print('=> Test') for name in target: print('Evaluating {} ...'.format(name)) queryloader = testloader_dict[name]['query'] galleryloader = testloader_dict[name]['gallery'] rank1, distmat = test(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank1) rank2, distmat2 = test_rerank(model, queryloader, galleryloader, test_batch_size, use_gpu) ranklogger.write(name, epoch + 1, rank2) del queryloader del galleryloader del distmat #print(torch.cuda.memory_allocated(),torch.cuda.memory_cached()) torch.cuda.empty_cache() if (epoch + 1) == max_epoch: #if (epoch + 1) % 10 == 0: print('=> Test') save_checkpoint( { 'state_dict': model.state_dict(), 'rank1': rank1, 'epoch': epoch + 1, 'arch': arch, 'optimizer': optimizer.state_dict(), }, save_dir) if vis_rank: visualize_ranked_results(distmat, dm.return_testdataset_by_name(name), save_dir=osp.join( save_dir, 'ranked_results', name), topk=20)
from datatype import Tensor def to_one_hot(vector: Tensor) -> Tensor: """Create one hot encoding of a vector.""" oh = np.zeros((vector.shape[0], vector.max() + 1)) oh[np.arange(vector.shape[0]), vector] = 1 return oh train = pd.read_csv("mnist_train.csv", header=None).values[:, 1:] train_label = pd.read_csv("mnist_train.csv", header=None).values[:, 0] net = Network(layers=[Linear(784, 128), Sigmoid(), Linear(128, 10),]) loss = CrossEntropyLoss() optim = SGD(5e-2, 0.001) x_train, x_val, y_train, y_val = train_test_split( train.astype(np.float32) / 255, train_label.astype(np.int32), test_size=0.2, random_state=42, ) # to_one_hot y_train = to_one_hot(y_train) y_val = to_one_hot(y_val) batch_size = 100 progress_bar = auto.tqdm(range(200)) accuracies: dict = {"train": [], "val": [], "test": []} acc_train: list = [] acc_val: list = []