def get_dataloaders(args): image_transform = transforms.Compose([ transforms.Resize((args.height, args.width)), transforms.ToTensor(), lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) mask_transform = transforms.Compose([ transforms.Resize((args.height, args.width)), transforms.ToTensor(), lambda m: torch.where(m > 0, torch.ones_like(m), torch.zeros_like(m)), lambda m: F.one_hot(m[0].to(torch.int64), 2).permute(2, 0, 1).to( torch.float32), ]) dl_args = { 'data_base_path': args.data_base_path, 'datasets': args.datasets.split(' '), 'image_transform': image_transform, 'mask_transform': mask_transform, 'batch_size': args.batch_size, } dl_train = get_dataloader(**dl_args, partition='train', shuffle=True) dl_validation = get_dataloader(**dl_args, partition='validation', shuffle=False) dl_test = get_dataloader(**dl_args, partition='test', shuffle=False) dls = { 'train': dl_train, 'validation': dl_validation, 'test': dl_test, } return dls
def main(args): seed = 2020 torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) # Make logdir if not os.path.exists(args.checkpoints_dir): os.makedirs(args.checkpoints_dir) # Load dataset train_dataloader = get_dataloader('train', args.bs, True, args.nw) val_dataloader = get_dataloader('val', args.bs, False, args.nw) # Model model = SimpleModel() optimizer = torch.optim.SGD( model.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.wd) model.cuda() train(args, train_dataloader, val_dataloader, model, optimizer)
def main(): args = get_parser().parse_args() print(args) # Make datasets train_dir = os.path.join(args.datadir, 'train') val_dir = os.path.join(args.datadir, 'val') print('loading train dataset') train_loader = get_dataloader(train_dir, args.batch_size, args.pretrained, args.augmented) print('loading val dataset') val_loader = get_dataloader(val_dir, args.batch_size, args.pretrained, False) args.num_class = 2 # np.unique(train_loader[1]) args.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Initialisation model model = Classifier(args=args) while model.counter['epochs'] < args.epochs: train(model=model, dataloader=train_loader) val(model=model, dataloader=val_loader) if model.early_stopping.early_stop: break if model.writer: model.writer.close()
def main(): # Prepare arguments opt = get_arguments().parse_args() if opt.dataset == "mnist" or opt.dataset == "cifar10": opt.num_classes = 10 elif opt.dataset == "gtsrb": opt.num_classes = 43 else: raise Exception("Invalid Dataset") if opt.dataset == "cifar10": opt.input_height = 32 opt.input_width = 32 opt.input_channel = 3 elif opt.dataset == "gtsrb": opt.input_height = 32 opt.input_width = 32 opt.input_channel = 3 elif opt.dataset == "mnist": opt.input_height = 28 opt.input_width = 28 opt.input_channel = 1 else: raise Exception("Invalid Dataset") # Load models and masks if opt.dataset == "cifar10": netC = PreActResNet18().to(opt.device) elif opt.dataset == "gtsrb": netC = PreActResNet18(num_classes=43).to(opt.device) elif opt.dataset == "mnist": netC = NetC_MNIST().to(opt.device) else: raise Exception("Invalid dataset") path_model = os.path.join( opt.checkpoints, opt.dataset, opt.attack_mode, "{}_{}_ckpt.pth.tar".format(opt.attack_mode, opt.dataset) ) state_dict = torch.load(path_model) print("load C") netC.load_state_dict(state_dict["netC"]) netC.to(opt.device) netC.eval() netC.requires_grad_(False) print("load G") netG = Generator(opt) netG.load_state_dict(state_dict["netG"]) netG.to(opt.device) netG.eval() netG.requires_grad_(False) print("load M") netM = Generator(opt, out_channels=1) netM.load_state_dict(state_dict["netM"]) netM.to(opt.device) netM.eval() netM.requires_grad_(False) # Prepare dataloader test_dl = get_dataloader(opt, train=False) test_dl2 = get_dataloader(opt, train=False) eval(netC, netG, netM, test_dl, test_dl2, opt)
def load_data(self, modes=["train", "valid", "test"]): self.train_dataloader = get_dataloader( self.config, mode="train") if "train" in modes else None self.valid_dataloader = get_dataloader( self.config, mode="valid") if "valid" in modes else None self.test_dataloader = get_dataloader( self.config, mode="test") if "test" in modes else None
def __init__(self): self.experiment_id = "PPF-FoldNet" + time.strftime('%m%d%H%M') snapshot_root = 'snapshot/%s' % self.experiment_id tensorboard_root = 'tensorboard/%s' % self.experiment_id os.makedirs(snapshot_root, exist_ok=True) os.makedirs(tensorboard_root, exist_ok=True) shutil.copy2(os.path.join('.', 'train.py'), os.path.join(snapshot_root, 'train.py')) shutil.copy2(os.path.join('.', 'models/model_conv1d.py'), os.path.join(snapshot_root, 'model.py')) self.epoch = 100 self.num_patches = 1 self.num_points_per_patch = 1024 # num of points per patches self.batch_size = 32 self.dataset = 'sun3d' self.data_train_dir = './data/3DMatch/rgbd_fragments' self.data_test_dir = './data/3DMatch/rgbd_fragments' self.gpu_mode = True self.verbose = True # model & optimizer self.model = PPFFoldNet(self.num_patches, self.num_points_per_patch) self.pretrain = '' self.parameter = self.model.get_parameter() self.optimizer = optim.Adam(self.parameter, lr=0.001, betas=(0.9, 0.999), weight_decay=1e-6) self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.5) self.scheduler_interval = 10 # dataloader self.train_loader = get_dataloader(root=self.data_train_dir, batch_size=self.batch_size, split='train', num_patches=self.num_patches, num_points_per_patch=self.num_points_per_patch, shuffle=True, num_workers=4, ) self.test_loader = get_dataloader(root=self.data_test_dir, batch_size=self.batch_size, split='test', num_patches=self.num_patches, num_points_per_patch=self.num_points_per_patch, shuffle=False, num_workers=4, ) print("Training set size:", self.train_loader.dataset.__len__()) print("Test set size:", self.test_loader.dataset.__len__()) # snapshot self.snapshot_interval = 100000 self.save_dir = os.path.join(snapshot_root, 'models/') self.result_dir = os.path.join(snapshot_root, 'results/') self.tboard_dir = tensorboard_root # evaluate self.evaluate_interval = 2 self.evaluate_metric = ChamferLoss() self.check_args()
def main(): global args, best_acc1, device criterion = PrototypicalLoss().to(device) cudnn.benchmark = True runs_path = glob('runs/*') max_len_exp = max([len(x) for x in runs_path]) + 2 print(f"|{'Experiment':^{max_len_exp}}|{'Loss':^17}|{'ACC':^17}|") except_list = [] pl_mi = u"\u00B1" for exp in glob('runs/*'): checkpoint, args = None, None files = glob(exp + '/*') for file in files: if file.endswith('model_best.pth'): checkpoint = torch.load(os.path.abspath(file)) elif file.endswith('.json'): params = json.load(open(os.path.abspath(file))) args = SimpleNamespace(**params) if checkpoint is None or args is None: except_list.append(f"checkpoint and params are not exist in {exp}") continue if args.dataset == 'omniglot': test_loader = get_dataloader(args, 'test') else: test_loader = get_dataloader(args, 'val') input_dim = 1 if args.dataset == 'omniglot' else 3 if args.model == 'protonet': model = ProtoNet(input_dim).to(device) else: model = ResNet(input_dim).to(device) model.load_state_dict(checkpoint['model_state_dict']) best_acc1 = checkpoint['best_acc1'] loss_list, acc_list = test(test_loader, model, criterion) loss, loss_moe = margin_of_error(loss_list) acc, acc_moe = margin_of_error(acc_list) loss_string = f'{loss:.3f} {pl_mi} {loss_moe:.3f}' acc_string = f'{acc:.3f} {pl_mi} {acc_moe:.3f}' print(f"|{exp:^{max_len_exp}}|{loss_string:^16}|{acc_string:^16}|") if len(except_list): pp(except_list)
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs): ckpt_dir = os.path.join(ckpt_path, '') ckpt_dir_flag = os.path.join(ckpt_dir, flag) prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl') dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl') os.makedirs(ckpt_dir, exist_ok=True) with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f: f.write('ckpt_dir') os.makedirs(ckpt_dir_flag, exist_ok=True) with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f: f.write('ckpt_dir_flag') file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) if os.path.exists(train_pkl): logger.info(f'=> load train, dev data from {ckpt_dir}') train_data = pk.load(open(train_pkl, 'rb')) # train_set is a of 1600 elements which are dict. # with keys:'mel' for mel-spectogram np array (80,xxx) and 'speaker' for speaker string ('p225') # print(f"**********************data is: {train_data[0]}********************") # print(f"**********************type is: {train_data[0]['mel'].shape}********************") dev_data = pk.load(open(dev_pkl, 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) # train_set is a class: dataloader.vctk.Dataset. # Has attribute data such that train_set.data is a list of 1600 features so train_set.data[i] is o size (80,xxx) # print(f"**********************type is: {type(train_set)}********************") # print(f"**********************shape is: {train_set.data[0]['mel'].shape}********************") dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) else: train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs) pk.dump(train_set.data, open(train_pkl, 'wb')) pk.dump(dev_set.data, open(dev_pkl, 'wb')) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) # print(f"**********************type is: {type(train_loader)}********************") return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def load_data(ckpt_path, dataset_config, dataloader_config, njobs): if os.path.isdir(ckpt_path): d = os.path.join(ckpt_path, '') with open(os.path.join(d, 'dirtype'), 'r') as f: dirtype = f.read().strip() if dirtype == 'ckpt_dir': logger.warn( f'The ckpt_path is {ckpt_path}, the flag is not specified.' ) logger.warn(f'Use "default" flag.') ckpt_dir = d flag = 'default' ckpt_dir_flag = os.path.join(ckpt_dir, flag) ckpt_path = ckpt_dir_flag elif dirtype == 'ckpt_dir_flag': ckpt_dir_flag = os.path.dirname(d) ckpt_dir = os.path.dirname(ckpt_dir_flag) flag = os.path.basename(ckpt_dir_flag) else: raise NotImplementedError( f'Wrong dirtype: {dirtype} from {d}.') else: ckpt_dir_flag = os.path.dirname(ckpt_path) ckpt_dir = os.path.dirname(ckpt_dir_flag) flag = os.path.basename(ckpt_dir_flag) file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) logger.info(f'=> load train, dev data from {os.path.join(ckpt_dir)}') prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_data = pk.load( open(os.path.join(ckpt_dir, f'{prefix}_train.pkl'), 'rb')) dev_data = pk.load( open(os.path.join(ckpt_dir, f'{prefix}_dev.pkl'), 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def __init__(self): self.experiment_id = "FoldNet" + time.strftime('%m%d%H%M') snapshot_root = 'snapshot/%s' % self.experiment_id tensorboard_root = 'tensorboard/%s' % self.experiment_id os.makedirs(snapshot_root, exist_ok=True) os.makedirs(tensorboard_root, exist_ok=True) shutil.copy2(os.path.join('.', 'train.py'), os.path.join(snapshot_root, 'train.py')) self.epoch = 300 self.num_points = 2048 self.batch_size = 16 self.dataset = 'shapenet' self.data_dir = 'data/shapenetcore_partanno_segmentation_benchmark_v0' self.gpu_mode = torch.cuda.is_available() self.verbose = False # model & optimizer self.model = FoldNet(self.num_points) self.pretrain = '' self.parameter = self.model.get_parameter() self.optimizer = optim.Adam(self.parameter, lr=0.001, betas=(0.9, 0.999), weight_decay=1e-6) self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.5) self.scheduler_interval = 100 # dataloader self.train_loader = get_dataloader(root=self.data_dir, split='train', classification=True, batch_size=self.batch_size, num_points=self.num_points, shuffle=True ) self.test_loader = get_dataloader(root=self.data_dir, split='test', classification=True, # if True then return pts & cls batch_size=self.batch_size, num_points=self.num_points, shuffle=False ) print("Training set size:", self.train_loader.dataset.__len__()) print("Test set size:", self.test_loader.dataset.__len__()) # snapshot self.snapshot_interval = 10 self.save_dir = os.path.join(snapshot_root, 'models/') self.result_dir = os.path.join(snapshot_root, 'results/') self.tboard_dir = tensorboard_root self.check_args()
def train(opt, init_mask, init_pattern): test_dataloader = get_dataloader(opt, train=False) # Build regression model regression_model = RegressionModel(opt, init_mask, init_pattern).to(opt.device) # Set optimizer optimizerR = torch.optim.Adam(regression_model.parameters(), lr=opt.lr, betas=(0.5, 0.9)) # Set recorder (for recording best result) recorder = Recorder(opt) for epoch in range(opt.epoch): early_stop = train_step(regression_model, optimizerR, test_dataloader, recorder, epoch, opt) if (early_stop): break # Save result to dir recorder.save_result_to_dir(opt) return recorder, opt
def evaluate(model, config, mode="test"): # load data dataloader = get_dataloader(config, mode=mode) # evaluate for i, data in enumerate(dataloader): pass
def main_fc(basetype, train=True, cluttered_translated=False): setup_dirs(data_dir='./data', ckpt_dir='./ckpt') torch.manual_seed(1) if torch.cuda.is_available(): torch.cuda.manual_seed(1) # get dataloaders batch_size = 20 if train else 1000 data_loader = dataloader.get_dataloader( data_dir='./data', batch_size=batch_size, random_seed=1, is_train=train, valid_size=0.1, shuffle=train, pin_memory=torch.cuda.is_available(), cluttered_translated=cluttered_translated) trainer = BaseTrainer(basetype, train, data_loader, cluttered_translated) if train: trainer.train() else: trainer.test() return
def load_data(C, logger): data_train, data_test, data_valid, relations, rel_weights = get_dataloader( C.dataset)( C, logger, C.train_text_1, C.train_rels_1, C.train_text_2, C.train_rels_2, C.test_text, C.test_rels, C.valid_text, C.valid_rels, C.dataset, C.rel_weight_smooth, C.rel_weight_norm, ) logger.log("num of sents / entity pairs in train: %d / %d" % (len(data_train), sum([len(x.ents) * (len(x.ents) - 1) / 2 for x in data_train]))) logger.log("num of sents / instances in test : %d / %d" % (len(data_test), sum([len(x.ans) for x in data_test]))) logger.log("num of sents / instances in valid : %d / %d" % (len(data_valid), sum([len(x.ans) for x in data_valid]))) logger.log("num of sents / entity pairs in test : %d / %d" % (len(data_test), sum([len(x.ents) * (len(x.ents) - 1) / 2 for x in data_test]))) logger.log("num of sents / entity pairs in valid : %d / %d" % (len(data_valid), sum([len(x.ents) * (len(x.ents) - 1) / 2 for x in data_valid]))) return data_train, data_test, data_valid, relations, rel_weights
def create_embeddings(config): exper_folder = config['dir'] embed_folder = config['embed_folder'] config_file = 'experiment/' + exper_folder + '/config.yaml' with open(config_file, 'r') as f: config = yaml.load(f) dataloader = get_dataloader(config, scope='test') model = get_model(config) model_weights = 'experiment/' + exper_folder + '/' + config['weights'] model.load_state_dict(torch.load(model_weights)['model']) if os.path.exists(embed_folder): shutil.rmtree(embed_folder) os.makedirs(embed_folder) with torch.no_grad(): for num_batch, sample in enumerate(dataloader): print('Batch {}/{} processing...'.format(num_batch + 1, len(dataloader))) sample['anchor'] = sample['anchor'].cuda(0) embeddings = model(sample['anchor']) embeddings = embeddings.detach().cpu().numpy() for num in range(len(sample['name'])): name = sample['name'][num].replace('/', '-') name = os.path.join(embed_folder, name) vector = embeddings[num] np.save(name, vector)
def test_process(self, best_model_file=None): if best_model_file is not None: self.best_model_file = best_model_file self.load_state_dict(torch.load(self.best_model_file)) test_loader = get_dataloader(self.config, mode="test", n_way=self.config.n_way, n_shot=self.config.n_shot, n_query=self.config.n_query_test) avg_acc = 0 avg_h = 0 best_acc = 0 best_h = 0 test_num = 10 for epoch in range(test_num): acc, h = self.test_loop(test_loader) self.logger.info("Test Acc = {:4.2f}+-{:4.2f}".format(acc, h)) if acc > best_acc: best_acc = acc best_h = h avg_acc += acc avg_h += h avg_acc /= test_num avg_h /= test_num self.logger.info("Best Test Acc = {:4.2f}+-{:4.2f}".format( best_acc, best_h)) self.logger.info("Avg Test Acc = {:4.2f}+-{:4.2f}".format( avg_acc, avg_h))
def main(config) -> None: talk_model = model.Model(config.model.vocab_size) talk_dataloader = dataloader.get_dataloader( config.data.data_category, config.train_conf.batch_size) loss = nn.CrossEntropyLoss() talk_trainer = trainer.Trainer(talk_model, talk_dataloader, loss, config.train_conf.optimizer, config.train_conf.optimizer_lr) talk_trainer.train(epoch=config.train_conf.epoch)
def inference_ocl_attr(model, current_epoch, current_iter, dataset_name, prev_metric_dict, seen_objects, finished_objects, all_objects, max_instance): """ :param model: :param current_epoch: :param current_iter: :param prev_metric_dict: {attr_acc: <attr>: [acc1, acc2]} :param filter_objects: :param filter_attrs: :return: """ model.train(False) device = torch.device('cuda') if not prev_metric_dict: prev_metric_dict = { 'attr_acc': defaultdict(list), 'inst_num': defaultdict(list), 'ft_dict': defaultdict(list), 'forget_dict': defaultdict(list), 'obj_acc': defaultdict(list), 'length': 0 } pbar = tqdm(total=len(all_objects), desc="Validation in progress") # only seen objects by this time for obj in all_objects: dataloader = get_dataloader(model.cfg, 'val', False, False, filter_obj=[obj]) obj_acc, attr_acc, inst_num = inference(model, current_epoch, current_iter, 0, dataloader, dataset_name, max_instance=max_instance, mute=True) prev_metric_dict['attr_acc'][obj].append(attr_acc) prev_metric_dict['inst_num'][obj].append(inst_num) prev_metric_dict['obj_acc'][obj].append(obj_acc) pbar.update(1) metric_dict = prev_metric_dict #run_forward_transfer_metrics(metric_dict, seen_objects, all_objects, 'attr_acc', metric_dict['ft_dict']) run_forget_metrics(metric_dict, finished_objects, all_objects, 'attr_acc', metric_dict['forget_dict']) metric_dict['length'] += 1 numerical_metric_dict = numericalize_metric_scores(metric_dict) return metric_dict, numerical_metric_dict
def main(): parser = argparse.ArgumentParser() parser.add_argument('--seq_embed_size', default=64) parser.add_argument('--k_hops', default=3) parser.add_argument('--hop_layers', default=3) parser.add_argument('--lstm_layers', default=3) parser.add_argument('--fn_layers', default=3) parser.add_argument('--lr', default=1e-4) parser.add_argument('--weight_decay', default=1e-8) parser.add_argument('--alpha', default=0.6) parser.add_argument('--epochs', default=50) parser.add_argument('--batch_size', default=2) parser.add_argument('--window_size', default=30) parser.add_argument('--device', default='cpu') args = parser.parse_args() if torch.cuda.is_available(): args.device = "cuda:0" # Load Data logger.info("Loading data...") nasdaq_data = pickle.load(open(NASDAQ_PICKLE_URL, "rb")) model_tickers = nasdaq_data.columns.tolist() # Load dataloaders & graphs inputs, dataloaders = get_dataloader(data=nasdaq_data, batch_size=args.batch_size, window_size=args.window_size, device=args.device) N, n_features = inputs train_loader, test_loader, val_loader = dataloaders encoding, _ = get_encodings('20180105', 'NASDAQ', model_tickers) relational_encoding = torch.FloatTensor(encoding).to(args.device) print("relational_encoding", relational_encoding.size()) # Initialize Model model = TemporalSAGE( input_shape=[args.batch_size, args.window_size, N, n_features], seq_embed_size=args.seq_embed_size, relational_encoding=relational_encoding, k_hops=args.k_hops, hop_layers=args.hop_layers, lstm_layers=args.lstm_layers, fn_layers=args.fn_layers, device=args.device) logger.info(f"\n{model}") logger.info(f"Starting training with args:") for arg in vars(args): logger.info(f"{arg}: {getattr(args, arg)}") train(model=model, train_loader=train_loader, val_loader=val_loader, args=args)
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs): ckpt_dir = os.path.join(ckpt_path, '') ckpt_dir_flag = os.path.join(ckpt_dir, flag) prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl') dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl') os.makedirs(ckpt_dir, exist_ok=True) with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f: f.write('ckpt_dir') os.makedirs(ckpt_dir_flag, exist_ok=True) with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f: f.write('ckpt_dir_flag') file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) if os.path.exists(train_pkl): logger.info(f'=> load train, dev data from {ckpt_dir}') train_data = pk.load(open(train_pkl, 'rb')) dev_data = pk.load(open(dev_pkl, 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) else: train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs) pk.dump(train_set.data, open(train_pkl, 'wb')) pk.dump(dev_set.data, open(dev_pkl, 'wb')) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def main(): feature_shape = (28, 28, 512) img_shape = (224, 224, 3) datagen = dataloader.get_dataloader('data', batch_size=4, shuffle=True) model = drn.drn_c_26(pretrained=True) generator = LabelGenerator(img_shape, feature_shape) result, images = generator.generate(datagen, model) print(result.shape) print(images.shape)
def main(): opt = get_arguments().parse_args() # prepare model classifier = MNIST_classifier() # prepare optimizer optimizer = torch.optim.SGD(classifier.parameters(), opt.lr, momentum=0.9) # prepare scheduler scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, opt.scheduler_milestones, opt.scheduler_lambda) # prepare dataloader dl_train = get_dataloader(opt, train=True) dl_test = get_dataloader(opt, train=False) # continue training ? create_dir(opt.checkpoint) path_model = os.path.join(opt.checkpoint, 'model_ckpt.pth.tar') if (os.path.exists(path_model)): print('Continue Training') state_dict = torch.load(path_model) classifier.load_state_dict(state_dict['classifier']) optimizer.load_state_dict(state_dict['optimizer']) scheduler.load_state_dict(state_dict['scheduler']) best_acc = state_dict['best_acc'] epoch = state_dict['epoch'] else: print('Train from scratch!!') best_acc = 0. epoch = 0 for epoch_idx in range(opt.n_iters): print('Epoch {}:'.format(epoch)) train(classifier, optimizer, scheduler, dl_train, opt) best_acc = evaluate(classifier, optimizer, scheduler, dl_test, best_acc, epoch, opt) epoch += 1
def train_process(self): train_loader = get_dataloader(self.config, mode="train", n_way=self.config.n_way, n_shot=self.config.n_shot, n_query=self.config.n_query_train) val_loader = get_dataloader(self.config, mode="val", n_way=self.config.n_way, n_shot=self.config.n_shot, n_query=self.config.n_query_test) for param in self.feature_extractor.layer3.block[ 0].conv_offset_mask.parameters(): param.requires_grad = False for param in self.feature_extractor.layer4.block[ 0].conv_offset_mask.parameters(): param.requires_grad = False for epoch in range(50): try: if epoch >= 5: for param in self.feature_extractor.layer3.block[ 0].conv_offset_mask.parameters(): param.requires_grad = True for param in self.feature_extractor.layer4.block[ 0].conv_offset_mask.parameters(): param.requires_grad = True self.train_loop(epoch, train_loader) self.val_loop(epoch, val_loader) except KeyboardInterrupt: if self.best_model_file == None: os.remove(self.logfile_name) else: print(Fore.YELLOW, end="") self.logger.info("KeyboardInterrupt!") self.post_process() print(Style.RESET_ALL, end="") exit()
def __init__(self, config): self.cuda = int(config['cuda']) self.train_dataloader = get_dataloader(config, scope='train') self.val_dataloader = get_dataloader(config, scope='val') self.model = get_model(config) try: model_weights = 'experiment/' + config['dir'] + '/' + config['weights'] self.model.load_state_dict(torch.load(model_weights)['model']) print('Weigths loaded') except: print('Weights randomized') self.optimizer = get_optimizer(config, self.model) self.total_epochs = config['epochs'] self.batches_per_epoch = config['batches_per_epoch'] self.val_batches_per_epoch = config['val_batches_per_epoch'] self.exp_name = config['dir'] self.save_weights_regularity = config['save_weights_regularity'] self.final_weights_file = 'experiment/' + config['dir'] + '/weights_last.pth' self.log_file = 'experiment/' + config['dir'] + '/logs.csv' self.loss = get_loss(config)
def main(args): # Load dataset test_dataloader = get_dataloader('test', args.bs, False, args.nw) # Model model = SimpleModel() model.cuda() ckpt = torch.load(os.path.join(args.checkpoints_dir, 'last_ckpt.pth')) model.load_state_dict(ckpt['model_state']) result = test(args, test_dataloader, model) # Make csv file df = pd.DataFrame({'id': test_dataloader.dataset.ids, 'category': result}) df.to_csv('out.csv', index=False)
def show_reconstructed(model, class_choice='Airplane'): dataroot = "data/shapenetcore_partanno_segmentation_benchmark_v0" dataloader = get_dataloader(root=dataroot, split='test', class_choice=class_choice, classification=True, num_points=2048, shuffle=False) pts, _ = dataloader.dataset[random.randint(0, 100)] reconstructed_pl = model(pts.view(1, 2048, 3))[0] ax1, _ = draw_pts(pts, clr=None, cmap='CMRmap') ax2, _ = draw_pts(reconstructed_pl.detach().numpy(), clr=None, cmap='CMRmap') ax1.figure.show() ax2.figure.show()
def run_topk_bottomk( k, scoredir, logdir, datadir, dataname, version, lam, loss, scale, iteration, save_path, device ): # load raw data raw = pkl.load(open(os.path.join(datadir, dataname),'rb')) # Load anomaly score dr_score = pd.read_csv(scoredir) _, _, testloader = get_dataloader(data_root=datadir, data_name=dataname, scale=scale, batch_size=1) # define window size and the number of features window_size = testloader.dataset.window_size nb_features = testloader.dataset.data.size(-1) # loss function if loss == 'mse': gen_criterion = torch.nn.MSELoss() elif loss == 'mae': gen_criterion = torch.nn.L1Loss() # Load saved model # logs 안에 version이 있어야함. version은 학습시 log 정보와 함께 생성됨 G_weights, _, _, _, _, best_metrics = load_model(resume=version, logdir=logdir) print('Best Loss: ',best_metrics) G = LSTMGenerator(nb_features = nb_features).to(device) G.load_state_dict(G_weights) # Visualization viz_topk_bottomk(k=k, anomaly_score=dr_score, G=G, dataloader=testloader, window_size=window_size, nb_features=nb_features, criterion=gen_criterion, iteration=iteration, colnames=raw.columns, device=device, save_path=save_path)
def evaluate_on_dataset(model, args): model.eval() model.cuda() dataloader = get_dataloader(args, is_train=True) for batch_nb, batch in enumerate(dataloader): rgb = batch['rgb'].cuda() skier = batch['skier'].cuda() flags = batch['flags'].cuda() rocks = batch['rocks'].cuda() trees = batch['trees'].cuda() gt_masks = torch.cat((skier, flags, rocks, trees), dim=1) pred_masks, latent = model(rgb) all_images = model.make_visuals(rgb, gt_masks, pred_masks) pass
def predict(model): criterion = nn.CrossEntropyLoss() _, test_loader = get_dataloader() pred_list = [] output_list = [] target_list = [] for data in test_loader: input = data['input'] target = data['target'] output = model(input) correct = torch.max(output.detach(), 1)[1] == target pred_list.append(sum(correct.numpy()) / len(correct.numpy())) output_list.append(output.detach().numpy()) target_list.append(target.detach().numpy()) print('Correct answer rate: {:.1f} %'.format( sum(pred_list) / len(pred_list) * 100))
def main(): model = TimeSeriesModel(n_bins, n_lag).to(device) optimizer = optim.Adam(model.parameters(), lr=learning_rate) scheduler = lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.97) if start_epoch != 0: checkpoint = torch.load( './log/checkpoint_epoch{}.pth'.format(start_epoch - 1)) model.load_state_dict(checkpoint['state_dict']) data_loaders, data_size = get_dataloader(batch_size=128, n_lag=n_lag) for epoch in range(start_epoch, num_epochs + start_epoch): print(80 * '=') print('Epoch [{}/{}]'.format(epoch, num_epochs + start_epoch - 1)) t0 = time.time() loss = train_valid(model, optimizer, scheduler, epoch, data_loaders, data_size) scheduler.step() t1 = time.time() #print("It took {0}s".format(t1-t0)) print(80 * '=')