def main(): config = get_config(mode='test') vocab = Vocab() vocab.load(config.word2id_path, config.id2word_path) print(f'Vocabulary size: {vocab.vocab_size}') config.vocab_size = vocab.vocab_size if config.users: test_users = load_pickle(config.convs_users_path) config.user_size = max([x for xx in test_users for x in xx]) + 1 print(f'User size: {config.user_size}') else: test_users = None data_loader = get_loader(convs=load_pickle(config.convs_path), convs_length=load_pickle(config.conversations_length_path), utterances_length=load_pickle(config.utterances_length_path), vocab=vocab, batch_size=config.batch_size, shuffle=False, convs_users=test_users) model_solver = getattr(solvers, "Solver{}".format(config.model)) test_solver = model_solver(config, None, data_loader, vocab=vocab, is_train=False) test_solver.build() test_solver.export_samples()
def do_tsne(options): options.shuffle = False options.batch_size = 2048 options.cuda_device = f"cuda:{get_gpu_ids()[0]}" # Load dataset into memory dataset = getattr(datasets, options.dataset)(options).test_set num = len(dataset) # Get the model model = get_model(options) proj_dim = options.projection_dim X = np.empty((num, proj_dim)) y = np.empty(num) # Get the dataloader loader = get_loader(dataset, options) idx = 0 for (batch, labels) in loader: num_batch = batch.shape[0] X[idx:idx + num_batch] = model(batch).cpu().numpy() y[idx:idx + num_batch] = labels.cpu().numpy() idx += num_batch X_new = TSNE().fit_transform(X) return (X_new, y)
def _eval_open_set_helper(self, discovered_samples, discovered_classes, roc_path, goscr_path, verbose=True, do_plot=True): dataset_loaders, _ = self.trainer_machine.get_trainloaders( discovered_samples, shuffle=False) train_loader = dataset_loaders['train'] test_loader = get_loader(self.dataset_info.test_dataset, None, shuffle=False, batch_size=self.batch, workers=self.workers) test_features = self._get_features(test_loader, self.trainer_machine, verbose=verbose).cpu() labeled_features = self._get_features(train_loader, self.trainer_machine, verbose=verbose).cpu() self.min_dist = self.distance_function( test_features, labeled_features).min(dim=1)[0].cpu().tolist() return super()._eval_open_set_helper(discovered_samples, discovered_classes, roc_path, goscr_path, verbose=verbose, do_plot=True)
def get_pre_classifier_pipeline(options, model): full_train_set = getattr(datasets, options.dataset)(options).plain_train_set options.batch_size = 2048 loader = get_loader(full_train_set, options) # standardize the scaler for all classifiers scaler = StandardScaler(copy=False) for batch, _ in loader: out = model(batch).detach().cpu().numpy() scaler.partial_fit(out) if options.model == "MoCoModel": steps = [('scaler', scaler)] elif options.model == "SelfLabelModel": if options.ipca: ipca = IncrementalPCA(copy=False, n_components=options.ipca_dim, batch_size=options.batch_size) for batch, _ in loader: out = model(batch).detach().cpu().numpy() ipca.partial_fit(scaler.transform(out)) steps = [('scaler', scaler), ('ipca', ipca)] else: steps = [('scaler', scaler)] elif options.model == "SimCLRModel": steps = [('scaler', scaler)] else: raise NotImplementedError( f"Classifier pipeline for {model} not implemented") pipeline = Pipeline(steps) options.pre_classifier_pipeline = pipeline return pipeline
def main(config): prepare_dirs_and_logger(config) rng = np.random.RandomState(config.random_seed) tf.set_random_seed(config.random_seed) if config.is_train: data_path = config.data_path batch_size = config.batch_size do_shuffle = True else: setattr(config, 'batch_size', 16) if config.test_data_path is None: data_path = config.data_path else: data_path = config.test_data_path batch_size = config.sample_per_image do_shuffle = False data_loader = get_loader(data_path, config.batch_size, config.input_scale_size, config.data_format, config.split) trainer = Trainer(config, data_loader) if config.is_train: save_config(config) trainer.train() else: if not config.load_path: raise Exception( "[!] You should specify `load_path` to load a pretrained model" ) trainer.test()
def main(args): model = utils.get_arch(arch=args.arch, dataset=args.dataset) criterion = torch.nn.CrossEntropyLoss() state_dict = torch.load(args.resume_path, map_location=torch.device('cpu')) model.load_state_dict( state_dict['model_state_dict'] ) del state_dict if not args.cpu: model.cuda() criterion = criterion.cuda() attacker = utils.PGDAttacker( radius = args.pgd_radius, steps = args.pgd_steps, step_size = args.pgd_step_size, random_start = True, norm_type = args.pgd_norm_type, ) if args.grad_opt == 'get-mu': ''' calculate grad_mu ''' train_loader = utils.get_loader( args.dataset, args.batch_size, train=True, training=False) grad_mu = get_grad_mu( model, criterion, train_loader, attacker, args.cpu) with open('{}/{}-mu.pkl'.format(args.save_dir, args.eval_save_name), 'wb') as f: pickle.dump(grad_mu, f) elif args.grad_opt == 'get-noise': ''' calculate grad noise ''' with open(args.resume_grad_mu_path, 'rb') as f: grad_mu = pickle.load(f) train_loader = utils.get_loader( args.dataset, args.batch_size, train=True, training=True) grad_noise = get_grad_noise( model, criterion, train_loader, attacker, grad_mu, args.grad_rep_T, args.grad_samp_T, args.cpu) with open('{}/{}-noise.pkl'.format(args.save_dir, args.eval_save_name), 'wb') as f: pickle.dump(grad_noise, f)
def main(): model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam( model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY, ) loss_fn = YoloLoss() scalar = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loader( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET+"/test.csv" ) if config.LOAD_MODEL: load_checkpoint( config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE, ) scaled_anchors = ( torch.tensor(config.ANCHORS) + torch.tensor(config.S).unsqueeze(1).unsqueeze(2).repeat(1, 3, 2) ).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): train_fn(test_loader, model, optimizer, loss_fn, scalar, scaled_anchors) if config.SAVE_MODEL: save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar") print(f"Currently epoch {epoch}") print("On Train Eval loader:") print("On Train Loader:") check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD) if epoch % 10 == 0 and epoch > 0: print("0n Test Loaded:") check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold = config.NMS_IOU_THRESH, anchors = config.ANCHORS, threshold = config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold = config.MAP_IOU_THRESH, box_format = "midpoint", num_classes = config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train()
def load_dataset(self, dataset_name, batch_size_train=100, batch_size_test=1000, val_ratio=0.3, n_cpu=8, transforms=None): """ Method that return the data loaders of a torchvision dataset :param dataset_name: name of the torchvision dataset batch_size_train: batch size to use for train data loader batch_size_test: batch size to use for test data loader val_ratio: percent of the data train to use for validation data loader n_cpu: num workers to use transforms: transformation to apply :return: train, validation and test torch data loader """ assert dataset_name == "CIFAR-10" or dataset_name == "CIFAR-100", \ "Pytorch datasets permitted are: CIFAR-10, CIFAR-100" if not os.path.exists(self.root_path): os.makedirs(self.root_path) train_dataset, test_dataset = None, None if dataset_name == "CIFAR-10": train_dataset = torchvision.datasets.CIFAR10( self.root_path, train=True, download=self.download, transform=transforms["train"]) test_dataset = torchvision.datasets.CIFAR10( self.root_path, train=False, download=self.download, transform=transforms["test"]) elif dataset_name == "CIFAR-100": train_dataset = torchvision.datasets.CIFAR100( self.root_path, train=True, download=self.download, transform=transforms["train"]) test_dataset = torchvision.datasets.CIFAR100( self.root_path, train=False, download=self.download, transform=transforms["test"]) train_loader, validation_loader, test_loader = utils.get_loader( train_dataset=train_dataset, test_dataset=test_dataset, batch_size_train=batch_size_train, batch_size_test=batch_size_test, n_cpu=n_cpu, val_ratio=val_ratio) return train_loader, validation_loader, test_loader
def generate(config, epoch): step_time = time.time() loader = utils.get_loader(config, config.sample_dir) sample_time = time.time() pool = utils.Image_Pool(config, config.sample_step, batch_size=1) print('-- Loading Images') for _, (image, _) in enumerate(loader): pool.add(image) print('-- Start Generating Images') idx = 0 count = 0 while True: images = pool() generated = net(variable(images)) start = (config.pool_size - config.sample_step) // 2 end = start + config.sample_step for i in range(start, end): image_a = images[0, :, i, :, :] image_b = generated.data[0, :, i, :, :].cpu() sample = torch.cat([image_a, image_b], dim=2) sample = utils.denorm(sample) if not os.path.isdir(config.result_dir): os.mkdir(config.result_dir) name = os.path.join(config.result_dir, 'Epoch-%d-%05d.jpg' % (epoch, count)) torchvision.utils.save_image(sample, name) count += 1 if not pool.check(): break if idx % config.log_frequency == 0: speed = (time.time() - step_time) / config.log_frequency step_time = time.time() format_str = 'Generating Images: Step %d Speed %.2f sec/frame' print(format_str % (idx, speed)) idx += 1
def main(config): prepare_dirs_and_logger(config) rng = np.random.RandomState(config.random_seed) tf.set_random_seed(config.random_seed) if config.is_train: data_path = config.data_path batch_size = config.batch_size do_shuffle=True else: setattr(config,'batch_size',64) if config.test_data_path is None: data_path = config.data_path else: data_path = config.test_data_path batch_size = config.sample_per_image do_shuffle=False if config.dataset == 'mnist' : mnist = tf.keras.datasets.mnist (x_train,y_train),(x_test,y_test) = mnist.load_data() data_loader = batch_generator(x_train, config.batch_size, config.data_format) elif config.dataset == 'cifar10': cifar = tf.keras.datasets.cifar10 (x_train,y_train),(x_test,y_test) = cifar.load_data() data_loader = batch_generator(x_train, config.batch_size, config.data_format) else: data_loader = get_loader( data_path, config.batch_size,config.scale_size, config.data_format) trainer = Trainer(config,data_loader) if config.is_train: save_config(config) trainer.train() else: if not config.load_path: raise Exception("[!] You should specify load_path to load a pretrained model") trainer.test()
def main(): start_time = time() args = get_args() if args.checkpoint_dir_name: dir_name = args.checkpoint_dir_name else: dir_name = datetime.datetime.now().strftime('%y%m%d%H%M%S') path_to_dir = Path(__file__).resolve().parents[1] path_to_dir = os.path.join(path_to_dir, *['log', dir_name]) os.makedirs(path_to_dir, exist_ok=True) # tensorboard path_to_tensorboard = os.path.join(path_to_dir, 'tensorboard') os.makedirs(path_to_tensorboard, exist_ok=True) writer = SummaryWriter(path_to_tensorboard) # model saving os.makedirs(os.path.join(path_to_dir, 'model'), exist_ok=True) path_to_model = os.path.join(path_to_dir, *['model', 'model.tar']) # csv os.makedirs(os.path.join(path_to_dir, 'csv'), exist_ok=True) path_to_results_csv = os.path.join(path_to_dir, *['csv', 'results.csv']) path_to_args_csv = os.path.join(path_to_dir, *['csv', 'args.csv']) if not args.checkpoint_dir_name: with open(path_to_args_csv, 'a') as f: args_dict = vars(args) param_writer = csv.DictWriter(f, list(args_dict.keys())) param_writer.writeheader() param_writer.writerow(args_dict) # logging using hyperdash if not args.no_hyperdash: from hyperdash import Experiment exp = Experiment('Classification task on CIFAR10 dataset with CNN') for key in vars(args).keys(): exec("args.%s = exp.param('%s', args.%s)" % (key, key, key)) else: exp = None path_to_dataset = os.path.join( Path(__file__).resolve().parents[2], 'datasets') os.makedirs(path_to_dataset, exist_ok=True) train_loader, eval_loader, classes = get_loader( batch_size=args.batch_size, num_workers=args.num_workers, path_to_dataset=path_to_dataset) # show some of the training images, for fun. dataiter = iter(train_loader) images, labels = dataiter.next() img_grid = torchvision.utils.make_grid(images) matplotlib_imshow(img_grid) writer.add_image('four_CIFAR10_images', img_grid) # define a network, loss function and optimizer model = CNN() writer.add_graph(model, images) model = torch.nn.DataParallel(model) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) start_epoch = 0 # resume training if args.checkpoint_dir_name: print('\nLoading the model...') checkpoint = torch.load(path_to_model) model.state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) start_epoch = checkpoint['epoch'] + 1 summary(model, input_size=(3, 32, 32)) model.to(args.device) # train the network print('\n--------------------') print('Start training and evaluating the CNN') for epoch in range(start_epoch, args.n_epoch): start_time_per_epoch = time() train_loss, train_acc = train(train_loader, model, criterion, optimizer, args.device, writer, epoch, classes) eval_loss, eval_acc = eval(eval_loader, model, criterion, args.device) elapsed_time_per_epoch = time() - start_time_per_epoch result_dict = { 'epoch': epoch, 'train_loss': train_loss, 'eval_loss': eval_loss, 'train_acc': train_acc, 'eval_acc': eval_acc, 'elapsed time': elapsed_time_per_epoch } with open(path_to_results_csv, 'a') as f: result_writer = csv.DictWriter(f, list(result_dict.keys())) if epoch == 0: result_writer.writeheader() result_writer.writerow(result_dict) # checkpoint torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, path_to_model) if exp: exp.metric('train loss', train_loss) exp.metric('eval loss', eval_loss) exp.metric('train acc', train_acc) exp.metric('eval acc', eval_acc) else: print(result_dict) writer.add_scalar('loss/train_loss', train_loss, epoch * len(train_loader)) writer.add_scalar('loss/eval_loss', eval_loss, epoch * len(eval_loader)) writer.add_scalar('acc/train_acc', train_acc, epoch * len(train_loader)) writer.add_scalar('acc/eval_acc', eval_acc, epoch * len(eval_loader)) elapsed_time = time() - start_time print('\nFinished Training, elapsed time ===> %f' % elapsed_time) if exp: exp.end() writer.close()
import torch import torch.nn.functional as F import os, sys, csv import yaml, time from model import Net from utils import get_loader with open(sys.argv[1]) as f: config = yaml.load(f, Loader=yaml.Loader) print('Preparing datasets...') loader, names = get_loader(if_label=False, imgs_folder=config['imgs_folder'], csv_file=config['csv_file'], resize_size=config['resize_size'], crop_size=config['crop_size'], is_train=config['is_train'], batch_size=config['batch_size'], num_workers=config['num_workers']) print('Constructing network...') net = Net(config) device = 'cuda:%d' % config['device'] if torch.cuda.is_available() else 'cpu' net.to(device) net.ready() print('Test start!') with torch.no_grad(): tot_pred = None for i, x in enumerate(loader): x = x.to(device) # x is 5D tensor
# 增加程序的运行速度 cudnn.benchmark = True # Create directories if not exist. if not os.path.exists(opts.log_path): os.makedirs(opts.log_path) if not os.path.exists(opts.model_save_dir): os.makedirs(opts.model_save_dir) if not os.path.exists(opts.output_path): os.makedirs(opts.output_path) if not os.path.exists(opts.result_dir): os.makedirs(opts.result_dir) # Load experiment setting config = get_config(opts.config) #为当前GPU设置随机种子,使得结果是确定的 torch.manual_seed(opts.seed) #为所有的GPU设置种子 torch.cuda.manual_seed(opts.seed) face_loader = None face_loader = get_loader(config['data_root'], config['face_crop_size'], config['face_image_size'], config['batch_size'], 'train', config['num_workers']) solver = Solver(face_loader, config, opts) if opts.mode == 'train': solver.train() elif opts.mode == 'test': solver.test()