def distance(X, Y, sqrt): cuda = torch.cuda.is_available() device = torch.device("cuda" if cuda else "cpu") data_loaderX = iter(utils.get_data_loader(X, len(X))) data_loaderY = iter(utils.get_data_loader(Y, len(Y))) X, _ = next(data_loaderX) Y, _ = next(data_loaderY) X, Y = X.to(device), Y.to(device) nX = X.size(0) nY = Y.size(0) X = X.view(nX, -1) X2 = (X * X).sum(1).resize_(nX, 1) Y = Y.view(nY, -1) Y2 = (Y * Y).sum(1).resize_(nY, 1) M = torch.zeros(nX, nY) M.copy_( X2.expand(nX, nY) + Y2.expand(nY, nX).transpose(0, 1) - 2 * torch.mm(X, Y.transpose(0, 1))) del X, X2, Y, Y2 if sqrt: M = ((M + M.abs()) / 2).sqrt() return M
def run(): # load dataset src_data_loader = get_data_loader(params.src_dataset) tgt_data_loader = get_data_loader(params.tgt_dataset) # load models src_encoder = init_model(net=LeNetEncoder(), restore=params.src_encoder_restore) tgt_encoder = init_model(net=LeNetEncoder(), restore=params.tgt_encoder_restore) critic = init_model(Discriminator(input_dims=params.d_input_dims, hidden_dims=params.d_hidden_dims, output_dims=params.d_output_dims), restore=params.d_model_restore) # Adapt target encoder by GAN print("=== Training encoder for target domain ===") print(">>> Target Encoder <<<") im, _ = next(iter(tgt_data_loader)) summary(tgt_encoder, input_size=im[0].size()) print(">>> Critic <<<") print(critic) # init weights of target encoder with those of source encoder if not tgt_encoder.restored: tgt_encoder.load_state_dict(src_encoder.state_dict()) # Train target if not (tgt_encoder.restored and critic.restored and params.tgt_model_trained): tgt_encoder = train_tgt(src_encoder, tgt_encoder, critic, src_data_loader, tgt_data_loader)
def run(): # load source dataset src_data_loader = get_data_loader(params.src_dataset) src_data_loader_eval = get_data_loader(params.src_dataset, train=False) # load models src_encoder = init_model(net=LeNetEncoder(), restore=params.src_encoder_restore) src_classifier = init_model(net=LeNetClassifier(), restore=params.src_classifier_restore) # pre-train source model print("=== Training classifier for source domain ===") print(">>> Source Encoder <<<") im, _ = next(iter(src_data_loader)) summary(src_encoder, input_size=im[0].size()) print(">>> Source Classifier <<<") print(src_classifier) if not (src_encoder.restored and src_classifier.restored and params.src_model_trained): src_encoder, src_classifier = train_src( src_encoder, src_classifier, src_data_loader) # eval source model print("=== Evaluating classifier for source domain ===") eval_src(src_encoder, src_classifier, src_data_loader_eval)
def get_dataset(args): # preprocess data print("=== Processing datasets ===") src_train = read_data('./data/processed/' + args.src + '/train.txt') src_test = read_data('./data/processed/' + args.src + '/test.txt') tgt_train = read_data('./data/processed/' + args.tgt + '/train.txt') tgt_test = read_data('./data/processed/' + args.tgt + '/test.txt') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') src_train_sequences = [] src_test_sequences = [] tgt_train_sequences = [] tgt_test_sequences = [] for i in range(len(src_train.review)): # 1587 tokenized_text = tokenizer.tokenize(src_train.review[i]) indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) src_train_sequences.append(indexed_tokens) for i in range(len(src_test.review)): tokenized_text = tokenizer.tokenize(src_test.review[i]) indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) src_test_sequences.append(indexed_tokens) for i in range(len(tgt_train.review)): tokenized_text = tokenizer.tokenize(tgt_train.review[i]) indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) tgt_train_sequences.append(indexed_tokens) for i in range(len(tgt_test.review)): tokenized_text = tokenizer.tokenize(tgt_test.review[i]) indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) tgt_test_sequences.append(indexed_tokens) # load dataset src_data_loader = get_data_loader(src_train_sequences, src_train.label, args) src_data_loader_eval = get_data_loader(src_test_sequences, src_test.label, args) tgt_data_loader = get_data_loader(tgt_train_sequences, tgt_train.label, args) tgt_data_loader_eval = get_data_loader(tgt_test_sequences, tgt_test.label, args) return src_data_loader, src_data_loader_eval, tgt_data_loader, tgt_data_loader_eval
def train(args, model, optimizer, scheduler=None, model_name='model'): # TODO: Q1.5 Initialize your visualizer here! # TODO: Q1.2 complete your dataloader in voc_dataset.py train_loader = utils.get_data_loader('voc', train=True, batch_size=args.batch_size, split='trainval') test_loader = utils.get_data_loader('voc', train=False, batch_size=args.test_batch_size, split='test') # Ensure model is in correct mode and on right device model.train() model = model.to(args.device) # TODO: Q1.4 Implement model training code! cnt = 0 for epoch in range(args.epochs): for batch_idx, (data, target, wgt) in enumerate(train_loader): # Get a batch of data data, target, wgt = data.to(args.device), target.to( args.device), wgt.to(args.device) optimizer.zero_grad() # Forward pass output = model(data) # Calculate the loss # TODO: your loss for multi-label clf? loss = 0 # Calculate gradient w.r.t the loss loss.backward() # Optimizer takes one step optimizer.step() # Log info if cnt % args.log_every == 0: # todo: add your visualization code print('Train Epoch: {} [{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, cnt, 100. * batch_idx / len(train_loader), loss.item())) # Validation iteration if cnt % args.val_every == 0: model.eval() ap, map = utils.eval_dataset_map(model, args.device, test_loader) model.train() cnt += 1 if scheduler is not None: scheduler.step() # Validation iteration test_loader = utils.get_data_loader('voc', train=False, batch_size=args.test_batch_size, split='test') ap, map = utils.eval_dataset_map(model, args.device, test_loader) return ap, map
def compute_loss(self, X, y, batch_size=1000): batch_size = min(batch_size, X.shape[0]) data_loader = utt.get_data_loader(X, y, batch_size, problem_type=self.problem_type) n = data_loader.sampler.num_samples # compute total loss total_loss = 0. for bi, (xb, yb) in enumerate(data_loader): if torch.cuda.is_available(): xb = xb.cuda() yb = yb.cuda() xb, yb = Variable(xb), Variable(yb) y_pred = self(xb) loss = lf.LOSS_DICT[self.loss_name](y_pred, yb) total_loss += loss.data[0] avg_loss = total_loss / float(n) #print "Average loss: %.3f" % avg_loss return avg_loss
def pretrain_root(root_model, model, pretrain_dataset, batch_iterations=1000, batch_size=32, n_classes=10): model.train() # Use cuda? cuda = model._is_on_cuda() device = model._device() iters_left = 1 active_classes = list(range(n_classes)) n_loads = 0 for batch_index in range(1, batch_iterations+1): iters_left -= 1 if iters_left==0: data_loader = iter(utils.get_data_loader(pretrain_dataset, batch_size, cuda=cuda, drop_last=True)) iters_left = len(data_loader) x, y = next(data_loader) #--> sample training data of current task n_loads += 1 x, y = x.to(device), y.to(device) #--> transfer them to correct device model.train_a_batch(x, y, active_classes=active_classes) # Transfer weights to root model_params = model.named_parameters() root_params = root_model.named_parameters() root_params_dict = dict(root_params) for name, param in model_params: if name in root_params_dict: root_params_dict[name].data.copy_(param.data)
def gatherStats(self, dataset, cuda_num=None, num_bits=8): stats = {} data_loader = iter( utils.get_data_loader(dataset, 64, cuda=True if cuda_num else False, drop_last=True)) iters_left = len(data_loader) device = torch.device("cuda:" + str(cuda_num) if cuda_num >= 0 else "cpu") with torch.no_grad(): for i in range(iters_left): data, target = next( data_loader) # --> sample training data of current task # data, target = data.to(device), target.to(device) stats = self.gatherActivationStats(data, stats) final_stats = {} for key, value in stats.items(): final_stats[key] = { "max": value["max"] / value["total"], "min": value["min"] / value["total"] } self.min_val = value["min"] / value["total"] self.max_val = value["max"] / value["total"] self.scale, self.zero_point = self.calcScaleZeroPoint( self.min_val, self.max_val, num_bits) return final_stats
def load_model(): dps = [] for i in range(params.provider_num): net = load_provider_model(i) dataloader = get_data_loader(i) dps.append(DataProvider(net, dataloader)) return dps
def folder2lmdb(dataset, outputFile, write_frequency=5000, num_workers=0 if os.name == 'nt' else 60): data_loader = get_data_loader(dataset, batch_size=1, num_workers=num_workers) print("Generate LMDB to %s" % outputFile) db = lmdb.open(outputFile, subdir=os.path.isdir(outputFile), map_size=10e+11, readonly=False, meminit=False, map_async=True) idx = 0 txn = db.begin(write=True) for audio, label in tqdm(data_loader): txn.put(u'{}'.format(idx).encode('ascii'), pickle.dumps( (audio, label))) idx += 1 if idx % write_frequency == 0: txn.commit() txn = db.begin(write=True) txn.commit() keys = [u'{}'.format(k).encode('ascii') for k in range(idx)] with db.begin(write=True) as txn: txn.put(b'__keys__', pickle.dumps(keys)) txn.put(b'__len__', pickle.dumps(len(keys))) print("Flushing database ...") db.sync() db.close()
def validate(model, dataset, batch_size=32, test_size=1024, verbose=True, allowed_classes=None, with_exemplars=False, no_task_mask=False, task=None): '''Evaluate precision (= accuracy or proportion correct) of a classifier ([model]) on [dataset]. [allowed_classes] None or <list> containing all "active classes" between which should be chosen (these "active classes" are assumed to be contiguous)''' # Set model to eval()-mode mode = model.training model.eval() # Apply task-specifc "gating-mask" for each hidden fully connected layer (or remove it!) if hasattr(model, "mask_dict") and model.mask_dict is not None: if no_task_mask: model.reset_XdGmask() else: model.apply_XdGmask(task=task) # Loop over batches in [dataset] data_loader = utils.get_data_loader(dataset, batch_size, cuda=model._is_on_cuda()) total_tested = total_correct = 0 for data, labels in data_loader: # -break on [test_size] (if "None", full dataset is used) if test_size: if total_tested >= test_size: break # -evaluate model (if requested, only on [allowed_classes]) data, labels = data.to(model._device()), labels.to(model._device()) #labels = labels - allowed_classes[0] if (allowed_classes is not None) else labels with torch.no_grad(): if with_exemplars: predicted = model.classify_with_exemplars( data, allowed_classes=allowed_classes) # - in case of Domain-IL scenario, collapse all corresponding domains into same class if max(predicted).item() >= model.classes: predicted = predicted % model.classes else: scores = model(data) if ( allowed_classes is None) else model(data)[:, allowed_classes] _, predicted = torch.max(scores, 1) # -update statistics total_correct += (predicted == labels).sum().item() total_tested += len(data) precision = total_correct / total_tested # Set model back to its initial mode, print result on screen (if requested) and return it model.train(mode=mode) if verbose: print('=> precision: {:.3f}'.format(precision)) return precision
def estimate_fisher(self, dataset, sample_size, batch_size=32): # sample loglikelihoods from the dataset. data_loader = utils.get_data_loader(dataset, batch_size) loglikelihoods = [] for x, y in data_loader: x = x.view(batch_size, -1) x = Variable(x).cuda() if self._is_on_cuda() else Variable(x) y = Variable(y).cuda() if self._is_on_cuda() else Variable(y) loglikelihoods.append( F.log_softmax(self(x), dim=1)[range(batch_size), y.data]) if len(loglikelihoods) >= sample_size // batch_size: break # estimate the fisher information of the parameters. loglikelihoods = torch.cat(loglikelihoods).unbind() loglikelihood_grads = zip(*[ autograd.grad( l, self.parameters(), retain_graph=(i < len(loglikelihoods))) for i, l in enumerate(loglikelihoods, 1) ]) loglikelihood_grads = [torch.stack(gs) for gs in loglikelihood_grads] fisher_diagonals = [(g**2).mean() for g in loglikelihood_grads] param_names = [ n.replace('.', '__') for n, p in self.named_parameters() ] return {n: f.detach() for n, f in zip(param_names, fisher_diagonals)}
def train(train_csv_path, model_path, batch_size, epochs, input_length, window_size): train_loader = utils.get_data_loader(train_csv_path, batch_size, True) model = models.PNN(input_length, window_size).to(DEVICE) model.apply(utils.init_normal) optimizer = optim.Adam(model.parameters()) train_model(model, epochs, train_loader, optimizer) torch.save(model.state_dict(), model_path)
def office(): init_random_seed(params.manual_seed) # load dataset src_data_loader = get_data_loader(params.src_dataset) src_data_loader_eval = get_data_loader(params.src_dataset, train=False) tgt_data_loader = get_data_loader(params.tgt_dataset) tgt_data_loader_eval = get_data_loader(params.tgt_dataset, train=False) # load models src_encoder = init_model(net=LeNetEncoder(), restore=params.src_encoder_restore) src_classifier = init_model(net=LeNetClassifier(), restore=params.src_classifier_restore) tgt_encoder = init_model(net=LeNetEncoder(), restore=params.tgt_encoder_restore) critic = init_model(Discriminator(input_dims=params.d_input_dims, hidden_dims=params.d_hidden_dims, output_dims=params.d_output_dims), restore=params.d_model_restore) if not (src_encoder.restored and src_classifier.restored and params.src_model_trained): src_encoder, src_classifier = train_src( src_encoder, src_classifier, src_data_loader) # eval source model # print("=== Evaluating classifier for source domain ===") # eval_src(src_encoder, src_classifier, src_data_loader_eval) # train target encoder by GAN # init weights of target encoder with those of source encoder if not tgt_encoder.restored: tgt_encoder.load_state_dict(src_encoder.state_dict()) if not (tgt_encoder.restored and critic.restored and params.tgt_model_trained): tgt_encoder = train_tgt(src_encoder, tgt_encoder, critic, src_data_loader, tgt_data_loader) # eval target encoder on test set of target dataset print(">>> domain adaption <<<") acc = eval_tgt(tgt_encoder, src_classifier, tgt_data_loader_eval) return acc
def main(): args = parse_arguments() n_vocab = params.n_vocab n_layer = params.n_layer n_hidden = params.n_hidden n_embed = params.n_embed n_batch = args.n_batch temperature = params.temperature train_path = params.train_path assert torch.cuda.is_available() print("loading_data...") # 训练时加载处理好的词典(如果有的话) if os.path.exists("vocab.json"): vocab = Vocabulary() with open('vocab.json', 'r') as fp: vocab.stoi = json.load(fp) for key, value in vocab.stoi.items(): vocab.itos.append(key) else: vocab = build_vocab(train_path, n_vocab) # save vocab with open('vocab.json', 'w') as fp: json.dump(vocab.stoi, fp) train_X, train_y, train_K = load_data(train_path, vocab) train_loader = get_data_loader(train_X, train_y, train_K, n_batch) print("successfully loaded") encoder = Encoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda() Kencoder = KnowledgeEncoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda() manager = Manager(n_hidden, n_vocab, temperature).cuda() decoder = Decoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda() if args.restore: encoder = init_model(encoder, restore=params.encoder_restore) Kencoder = init_model(Kencoder, restore=params.Kencoder_restore) manager = init_model(manager, restore=params.manager_restore) decoder = init_model(decoder, restore=params.decoder_restore) # ToDo:目前的代码所有的embedding都是独立的,可以参考transformer源码使用直接赋值的方法共享参数: #if emb_src_trg_weight_sharing: # self.encoder.src_word_emb.weight = self.decoder.trg_word_emb.weight model = [encoder, Kencoder, manager, decoder] parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \ list(manager.parameters()) + list(decoder.parameters()) optimizer = optim.Adam(parameters, lr=args.lr) # pre_train knowledge manager print("start pre-training") pre_train(model, optimizer, train_loader, args) print("start training") train(model, optimizer, train_loader, args) # save final model save_models(model, params.all_restore)
def predict(predict_csv_path, output_csv_path, model_path, batch_size, input_length, window_size): predict_loader, file_path_list = utils.get_data_loader(predict_csv_path, batch_size, False, True) model = models.PNN(input_length, window_size).to(DEVICE) model.load_state_dict(torch.load(model_path)) predictions = predict_model(model, predict_loader) with open(output_csv_path, 'w') as f: for fn, prediction in zip(file_path_list, predictions): f.write('{},{}\n'.format(fn, prediction))
def _run_train(self, train_dataset, iters, batch_size, loss_cbs, target_transform, replayed_dataset=None): iters_left = 1 cuda = self._is_on_cuda() device = self._device() for idx, c in enumerate(train_dataset.classes, 0): singlelabel_dataset = train_dataset.filter([idx]) singlelabel_dataset.set_target_tranform(target_transform) class_index = target_transform(c) progress = tqdm.tqdm(range(1, iters + 1)) for batch_index in range(1, iters + 1): iters_left -= 1 if iters_left == 0: data_loader = iter( utils.get_data_loader(singlelabel_dataset, batch_size, cuda=cuda, drop_last=True)) iters_left = len(data_loader) x, y = next( data_loader) #--> sample training data of current task x, y = x.to(device), y.to( device) #--> transfer them to correct devi scores = None if batch_index <= iters: # Train the main model with this batch instance_noise_factor = 0 if self.noisy: instance_noise_factor = ((iters - batch_index) * 1.0 / iters) loss_dict = self.train_a_batch(x, y, class_index=class_index, noise=instance_noise_factor) for loss_cb in loss_cbs: if loss_cb is not None: loss_cb(progress, batch_index, loss_dict, task=class_index) # Close progres-bar(s) progress.close()
def cal_val_acc(net): net = net.to(device) net.eval() mapping = np.load(os.path.join('preproc_data','map.npz'))['map'].reshape(1)[0] criterion = nn.CrossEntropyLoss() x_val, y_val = utils.read_preproc_data(os.path.join('preproc_data', 'val.npz')) val_loader = utils.get_data_loader(x_val, y_val, mapping, data_aug = False, batch_size = 32, shuffle = False) val_acc = valid(net, criterion, val_loader)
def show_reconstruction(model, dataset, config, pdf=None, visdom=None, size=32, task=None, collate_fn=None): '''Plot reconstructed examples by an auto-encoder [model] on [dataset], in [pdf] and/or in [visdom].''' # Set model to evaluation-mode mode = model.training model.eval() # Get data data_loader = utils.get_data_loader(dataset, size, shuffle=False, cuda=model._is_on_cuda(), collate_fn=collate_fn) (data, labels) = next(iter(data_loader)) data, labels = data.to(model._device()), labels.to(model._device()) # Evaluate model with torch.no_grad(): recon_batch, y_hat, mu, logvar, z = model(data, full=True) # Plot original and reconstructed images comparison = torch.cat([ data.view(-1, config['channels'], config['size'], config['size'])[:size], recon_batch.view(-1, config['channels'], config['size'], config['size'])[:size] ]).cpu() image_tensor = comparison.view(-1, config['channels'], config['size'], config['size']) # -number of rows nrow = int(np.ceil(np.sqrt(size * 2))) # -make plots if pdf is not None: task_stm = "" if task is None else " (task {})".format(task) visual_plt.plot_images_from_tensor(image_tensor, pdf, nrow=nrow, title="Reconstructions" + task_stm) if visdom is not None: visual_visdom.visualize_images( tensor=image_tensor, name='Reconstructions ({})'.format(visdom["graph"]), env=visdom["env"], nrow=nrow, ) # Set model back to initial mode model.train(mode=mode)
def main(): # model = models.resnet18(pretrained=False) # model.fc = nn.Linear(512,20) # model.load_state_dict(torch.load('/home/biorobotics/VLR/Visual-Learning-Recognition-16824/HW1/q4_resnet18_models_v1/model_at_epoch_9.pth')) # model = model.to(device) # model.eval() # test_loader, index_list = utils.get_data_loader('voc', train=False, batch_size=args.test_batch_size, split='test') # print(len(index_list)) # img_idx = np.random.choice(len(test_loader),size=3,replace=False) _, index_list = utils.get_data_loader('voc', train=False, batch_size=args.test_batch_size, split='test') index_list = np.array(index_list) # import ipdb; ipdb.set_trace() img1_idx = np.where(index_list=='000002')[0][0] # 000002.jpg train img2_idx = np.where(index_list=='000029')[0][0] # 000029.jpg # dog img3_idx = np.where(index_list=='000135')[0][0] # 000135.jpg # car # img1_idx = 1 # print(img1_idx) response_list = [] # for batch_idx, (data, target, wgt) in enumerate(test_loader): # print(batch_idx) # data = data.to(device) # model.avgpool.register_forward_hook(get_layer_response('avgpool')) # out = model(data) # response_list.append(activation['avgpool']) # print(len(response_list)) # with open("/home/biorobotics/VLR/Visual-Learning-Recognition-16824/HW1/resnet_feat.txt", "wb") as file: # pickle.dump(response_list, file) with open("/home/biorobotics/VLR/Visual-Learning-Recognition-16824/HW1/resnet_feat.txt", "rb") as file: response_list = pickle.load(file) # print(response_list[0].view(-1)) response_list_2d = [] for i in range(len(response_list)): # print(i) response_list_2d.append(response_list[i].view(-1).detach().cpu().numpy()) # print(np.shape(response_list_2d)) neigh = NearestNeighbors(n_neighbors=4) neigh.fit(np.array(response_list_2d)) # import ipdb; ipdb.set_trace() closest_to_img1 = neigh.kneighbors(np.array(response_list_2d[img1_idx]).reshape(1,-1)) closest_to_img2 = neigh.kneighbors(np.array(response_list_2d[img2_idx]).reshape(1,-1)) closest_to_img3 = neigh.kneighbors(np.array(response_list_2d[img3_idx]).reshape(1,-1)) print("Input Image Num: ", index_list[img1_idx], " Output image nums: ", index_list[closest_to_img1[1][0,0]], index_list[closest_to_img1[1][0,1]], index_list[closest_to_img1[1][0,2]], index_list[closest_to_img1[1][0,3]]) print("Input Image Num: ", index_list[img2_idx], " Output image nums: ", index_list[closest_to_img2[1][0,0]], index_list[closest_to_img2[1][0,1]], index_list[closest_to_img2[1][0,2]], index_list[closest_to_img2[1][0,3]]) print("Input Image Num: ", index_list[img3_idx], " Output image nums: ", index_list[closest_to_img3[1][0,0]], index_list[closest_to_img3[1][0,1]], index_list[closest_to_img3[1][0,2]], index_list[closest_to_img3[1][0,3]])
def main(): global args, config, last_epoch, best_prec, writer writer = SummaryWriter(log_dir=args.work_path + '/event') # 加载配置文件 with open(args.work_path + '/config.yaml') as f: config = yaml.load(f) config = easydict.EasyDict(config) logger.info((config)) # 获取模型 net = get_model(config) logger.info(net) logger.info("=====total parameters:" + str(utils.count_parameters(net))) device = 'cuda' if config.use_gpu else 'cpu' if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True net.to(device) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), config.lr_scheduler.base_lr, momentum=config.optimize.momentum, weight_decay=config.optimize.weight_decay, nesterov=config.optimize.nesterov) last_epoch = -1 best_prec = 0 # 加载训练过的模型继续训练 if args.work_path: ckpt_file_name = args.work_path + '/' + config.ckpt_name + '.pth.tar' if args.resume: best_prec, last_epoch = utils.load_checkpoint(ckpt_file_name, net, optimizer=optimizer) # 设置数据的格式转换 transform_train = transforms.Compose(utils.data_augmentation(config)) transform_test = transforms.Compose( utils.data_augmentation(config, is_train=False)) train_loader, test_loader = utils.get_data_loader(transform_train, transform_test, config) logger.info("==============trian-test-file-pathL{}".format(config.dataset)) logger.info(" ======= Training =======\n") for epoch in range(last_epoch + 1, config.epochs): lr = utils.adjust_learning_rate(optimizer, epoch, config) writer.add_scalar('learning_rate', lr, epoch) train(train_loader, net, criterion, optimizer, epoch, device) if epoch == 0 or ( epoch + 1) % config.eval_freq == 0 or epoch == config.epochs - 1: test(test_loader, net, criterion, optimizer, epoch, device) writer.close() logger.info( "======== Training Finished. best_test_acc: {:.3f}% ========".format( best_prec))
def __init__(self, model: nn.Module, criterion: nn.Module, dataset_name: str, monitor_cls: type = Monitor): self.model = model self.criterion = criterion self.dataset_name = dataset_name self.train_loader = get_data_loader(dataset_name, train=True) self.monitor = monitor_cls(self) self._monitor_parameters(self.model) self.volatile = False
def prepare_data(train=True, onehot=False, take_first=None): loader = get_data_loader(dataset="MNIST", train=train) x_data, y_data = flatten_dataset(data_loader=loader, cast_numpy=True, take_first=take_first) x_data = (x_data > 0).astype(np.float32) if onehot: n_samples = len(y_data) labels_onehot = np.zeros(shape=(n_samples, 10), dtype=np.int8) labels_onehot[range(n_samples), y_data] = 1 y_data = labels_onehot return x_data, y_data
def validate(model, dataset, batch_size=128, test_size=1024, verbose=True, allowed_classes=None, no_task_mask=False, task=None): '''Evaluate precision (= accuracy or proportion correct) of a classifier ([model]) on [dataset]. [allowed_classes] None or <list> containing all "active classes" between which should be chosen (these "active classes" are assumed to be contiguous)''' # Get device-type / using cuda? device = model._device() cuda = model._is_on_cuda() # Set model to eval()-mode model.eval() # Apply task-specifc "gating-mask" for each hidden fully connected layer (or remove it!) if model.mask_dict is not None: if no_task_mask: model.reset_XdGmask() else: model.apply_XdGmask(task=task) # Loop over batches in [dataset] data_loader = utils.get_data_loader(dataset, batch_size, cuda=cuda) total_tested = total_correct = 0 for data, labels in data_loader: # -break on [test_size] (if "None", full dataset is used) if test_size: if total_tested >= test_size: break # -evaluate model (if requested, only on [allowed_classes]) data, labels = data.to(device), labels.to(device) labels = labels - allowed_classes[0] if (allowed_classes is not None) else labels with torch.no_grad(): scores = model.classify(data, not_hidden=True) scores = scores if ( allowed_classes is None) else scores[:, allowed_classes] _, predicted = torch.max(scores, 1) # -update statistics total_correct += (predicted == labels).sum().item() total_tested += len(data) precision = total_correct / total_tested # Print result on screen (if requested) and return it if verbose: print('=> precision: {:.3f}'.format(precision)) return precision
def show_reconstruction(model, dataset, config, pdf=None, visdom=None, size=32, task=None, collate_fn=None): '''Plot reconstructed examples by an auto-encoder [model] on [dataset], in [pdf] and/or in [visdom].''' cuda = model._is_on_cuda() # Set model to evaluation-mode mode = model.training model.eval() # Get data data_loader = utils.get_data_loader(dataset, size, cuda=cuda, collate_fn=collate_fn) (data, labels) = next(iter(data_loader)) # Evaluate model data = Variable(data, volatile=True).cuda() if cuda else Variable( data, volatile=True) recon_batch, y_hat, mu, logvar, z = model(data, full=True) # Plot original and reconstructed images comparison = torch.cat([ data.view(-1, config['channels'], config['size'], config['size'])[:size], recon_batch.view(-1, config['channels'], config['size'], config['size'])[:size] ]).cpu() image_tensor = comparison.data.view(-1, config['channels'], config['size'], config['size']) if pdf is not None: task_stm = "" if task is None else " (task {})".format(task) visual_plt.plot_images_from_tensor(image_tensor, pdf, nrow=8, title="Reconstructions" + task_stm) if visdom is not None: visual_visdom.visualize_images( tensor=image_tensor, name='reconstructed samples ({})'.format(visdom["graph"]), env=visdom["env"], ) # Set model back to initial mode model.train(mode=mode)
def validate(model, dataset, batch_size=128, test_size=1024, verbose=True, collate_fn=None, allowed_classes=None, task_mask=False, task=None): '''Evaluate precision (= accuracy or proportion correct) of a classifier ([model]) on [dataset]. [allowed_classes] None or <list> containing all "active classes" between which should be chosen (these "active classes" are assumed to be contiguous)''' # Set model to eval()-mode mode = model.training model.eval() # Apply task-specifc "gating-mask" for each hidden fully connected layer if task_mask: model.apply_XdGmask(task=task) # Loop over batches in [dataset] data_loader = utils.get_data_loader(dataset, batch_size, cuda=model._is_on_cuda(), collate_fn=collate_fn) total_tested = total_correct = 0 for data, labels in data_loader: # -break on [test_size] (if "None", full dataset is used) if test_size: if total_tested >= test_size: break # -evaluate model (if requested, only on [allowed_classes]) data = Variable(data).cuda() if model._is_on_cuda() else Variable(data) labels = Variable(labels).cuda() if model._is_on_cuda() else Variable( labels) if (allowed_classes is not None): labels = labels - allowed_classes[0] scores = model(data) if ( allowed_classes is None) else model(data)[:, allowed_classes] _, predicted = torch.max(scores, 1) # -update statistics total_correct += (predicted == labels).sum().data[0] total_tested += len(data) precision = total_correct / total_tested # Set model back to its initial mode, print result on screen (if requested) and return it model.train(mode=mode) if verbose: print('=> precision: {:.3f}'.format(precision)) return precision
def main(): args = parser.parse_args() config = vars(args) train_loader,val_loader,test_loader = get_data_loader(dataset_name=config['dataset'],\ data_path=config['dataset_path'],\ TRAIN_BATCH_SIZE=config['train_batch_size'],\ VAL_BATCH_SIZE=config['val_batch_size'],\ TEST_BATCH_SIZE=config['test_batch_size']) model = get_network(config['network']) model.train() model.cuda() train_reg2(model, train_loader, config) evaluate(model, val_loader, test_loader, config)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, help='Path to the config file') parser.add_argument('--resume', action='store_true') parser.add_argument('--gpu_id', type=int, default=0, help='gpu_id') opts = parser.parse_args() cudnn.benchmark = True # the code can accelerate the training usually torch.cuda.set_device(opts.gpu_id) # Load experiment setting config = utils.get_config(opts.config) max_iter = config['max_iter'] # setup model and data loader pretrainer = EncoderTrainer(config, ispretrain=True) pretrainer.cuda() pretrainer.eval() # close BN and droplet trainloader, testloader = utils.get_data_loader(config, isPretrain=True) # setup logger and output folders model_name = os.path.splitext(os.path.basename(opts.config))[0] model_dir = os.path.join(config['output_pth'], model_name) log_dir, checkpoint_dir, image_dir, test_dir = utils.prepare_folder_strcutre( model_dir) # start test iterations = pretrainer.resume(checkpoint_dir, config) test_log_pth = os.path.join(test_dir, '%d.txt' % iterations) num_data, avg_loss = pre_sample(pretrainer, trainloader, config['batch_size']) msg = [ 'The train dataset has %d input data, ' % num_data + 'For pretrain-model%d-%d,the avarage vec_rec_loss is %f\n' % (config['input_option'], iterations, avg_loss) ] num_data, avg_loss = pre_sample(pretrainer, testloader, config['batch_size']) msg.append('The test dataset has %d input data, ' % num_data + 'For pretrain-model%d-%d,the avarage vec_rec_loss is %f\n' % (config['input_option'], iterations, avg_loss)) for m in msg: print m # write log with open(test_log_pth, 'w') as f: f.writelines(msg)
def train(train_csv_path, model_path, batch_size, epochs): try: train_loader = utils.get_data_loader(train_csv_path, batch_size, True) except Exception as e: print(e) sys.exit(1) model = Invincea.Invincea().to(DEVICE) optimizer = optim.Adam(model.parameters()) if model_path == None: model_path = 'model.dat' if os.path.isfile(model_path): model.load_state_dict(torch.load(model_path)) train_loader(model, epochs, train_loader, optimizer) torch.save(model.state_dict(), model_path)
def nearest_neighbors(): # PATH = "./checkpoints/resnet_pretrained.pth" # model = models.resnet18() # model.fc = nn.Linear(in_features=512, out_features=len(VOCDataset.CLASS_NAMES), bias=True) PATH = "./checkpoints/caffenet.pth" model = CaffeNet() model = CaffeNet(num_classes=len(VOCDataset.CLASS_NAMES), inp_size=227, c_dim=3).to(device) model.load_state_dict( torch.load(PATH, map_location=lambda storage, loc: storage)) model.eval() model = model.to(device) test_loader = utils.get_data_loader('voc', train=False, batch_size=args.test_batch_size, split='test') model.max_pool5.register_forward_hook(feature_append) dataset = VOCDataset('test', 64) with torch.no_grad(): for data, target, wgt in test_loader: data, target, wgt = data.to(device), target.to(device), wgt.to( device) output = model(data) pool_features = np.vstack(pool_features_list) nbrs = NearestNeighbors(n_neighbors=5, algorithm='ball_tree').fit(pool_features) distances, indices = nbrs.kneighbors(pool_features) output_no = 0 no_neighbors = 5 selected_class_idx = [] for i in range(pool_features.shape[0]): rand_idx = np.random.randint(low=0, high=pool_features.shape[0]) class_label, _ = dataset.anno_list[rand_idx] class_label = np.squeeze(np.argwhere(class_label == 1)).tolist() if len(list(set(selected_class_idx) & set(class_label))) == 0: selected_class_idx.extend(class_label) for i in range(no_neighbors): vis_image(dataset, indices[rand_idx, i]) import pdb pdb.set_trace()