def evaluate(args): label_map = load_label_map(args.dataset) n_classes = 50 if args.dataset == "include": n_classes = 263 if args.use_cnn: dataset = FeaturesDatset( features_dir=os.path.join(args.data_dir, f"{args.dataset}_test_features"), label_map=label_map, mode="test", ) else: dataset = KeypointsDataset( keypoints_dir=os.path.join(args.data_dir, f"{args.dataset}_test_keypoints"), use_augs=False, label_map=label_map, mode="test", max_frame_len=169, ) dataloader = data.DataLoader( dataset, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True, ) if args.model == "lstm": config = LstmConfig() if args.use_cnn: config.input_size = CnnConfig.output_dim model = LSTM(config=config, n_classes=n_classes) else: config = TransformerConfig(size=args.transformer_size) if args.use_cnn: config.input_size = CnnConfig.output_dim model = Transformer(config=config, n_classes=n_classes) model = model.to(device) if args.use_pretrained == "evaluate": model, _, _ = load_pretrained(args, n_classes, model) print("### Model loaded ###") else: exp_name = get_experiment_name(args) model_path = os.path.join(args.save_path, exp_name) + ".pth" ckpt = torch.load(model_path) model.load_state_dict(ckpt["model"]) print("### Model loaded ###") test_loss, test_acc = validate(dataloader, model, device) print("Evaluation Results:") print(f"Loss: {test_loss}, Accuracy: {test_acc}")
def create_distance_matrix(dataset): experiment_name = get_experiment_name(EXPERIMENT_NAME, pairwise=True, ranking=True, adaptive=ADAPTIVE, loss=LOSS) dataset = list(dataset) net = FinalResnet() # if pretrained_model_name is not None: # checkpoint_path = get_checkpoint_path(pretrained_model_name) # data = torch.load(checkpoint_path, map_location=DEVICE) # else: # data = None adver_net = AdverserialNetwork() stats_manager = StatsManager() exp = AdaptiveExperiment(net, adver_net, stats_manager, output_dir=experiment_name, perform_validation_during_training=False, pretrained_data=None) ref_img1, ref_img2 = get_two_ref_target_imgs() dataset.extend([ref_img1[0].unsqueeze(0)] + [ref_img2[0].unsqueeze(0)]) num_samples = len(dataset) distance_matrix = np.zeros((num_samples, num_samples)) exp.net.eval() exp.adv_net.eval() with torch.no_grad(): for i in range(num_samples - 1): for j in range(i + 1, num_samples): x1_s, x2_s = dataset[i], dataset[j] if isinstance(x1_s, np.ndarray) and isinstance( x2_s, np.ndarray): x1_s = torch.from_numpy(x1_s) x2_s = torch.from_numpy(x2_s) x = torch.cat([x1_s, x2_s], dim=1) x = x.to(exp.net.device) f, y = exp.net.forward_adaptive(x) distance_matrix[i, j] = y[0][0] distance_matrix[j, i] = y[0][0] return distance_matrix, ref_img1[1], ref_img2[1]
def evaluate(args): test_files = sorted( glob.glob( os.path.join(args.data_dir, f"{args.dataset}_test_keypoints", "*.json"))) test_df = load_dataframe(test_files) label_map = load_label_map(args.dataset) x_test, y_test = preprocess(test_df, args.use_augs, label_map, "test") exp_name = get_experiment_name(args) config = XgbConfig() model = Xgboost(config=config) load_path = os.path.join(args.save_dir, exp_name, ".pickle.dat") model.load(load_path) print("### Model loaded ###") test_preds = model(x_test) print("Test accuracy:", accuracy_score(y_test, test_preds))
def fit(args): train_files = sorted( glob.glob( os.path.join(args.data_dir, f"{args.dataset}_train_keypoints", "*.json"))) val_files = sorted( glob.glob( os.path.join(args.data_dir, f"{args.dataset}_val_keypoints", "*.json"))) train_df = load_dataframe(train_files) val_df = load_dataframe(val_files) label_map = load_label_map(args.dataset) x_train, y_train = preprocess(train_df, args.use_augs, label_map, "train") x_val, y_val = preprocess(val_df, args.use_augs, label_map, "val") config = XgbConfig() model = Xgboost(config=config) model.fit(x_train, y_train, x_val, y_val) exp_name = get_experiment_name(args) save_path = os.path.join(args.save_dir, exp_name, ".pickle.dat") model.save(save_path)
from data import get_train_loader, get_test_loaders from losses import * from continuous_losses import * from networks import initialize_model, freeze_layers, freeze_conv_layers from train import train, continuous_train from metrics import initialize_metrics, evaluation, update_metrics, write_results import utils from utils import device args = utils.config() print(args) experiment_name = utils.get_experiment_name() utils.make_directory("../logs") logging.basicConfig(filename=os.path.join( '../logs/{}.log'.format(experiment_name)), level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s %(message)s') logger = logging.getLogger(__name__) utils.print_log('START with Configuration : {}'.format(args)) data_path = utils.make_directory(args.data_path) partitions, partitions_train, partitions_tune = utils.get_partitions() if args.dataset == "Cars3D": train_data = Cars3D(root=args.data_path, mode="train",
def fit(args): exp_name = get_experiment_name(args) logging_path = os.path.join(args.save_path, exp_name) + ".log" logging.basicConfig(filename=logging_path, level=logging.INFO, format="%(message)s") seed_everything(args.seed) label_map = load_label_map(args.dataset) if args.use_cnn: train_dataset = FeaturesDatset( features_dir=os.path.join(args.data_dir, f"{args.dataset}_train_features"), label_map=label_map, mode="train", ) val_dataset = FeaturesDatset( features_dir=os.path.join(args.data_dir, f"{args.dataset}_val_features"), label_map=label_map, mode="val", ) else: train_dataset = KeypointsDataset( keypoints_dir=os.path.join(args.data_dir, f"{args.dataset}_train_keypoints"), use_augs=args.use_augs, label_map=label_map, mode="train", max_frame_len=169, ) val_dataset = KeypointsDataset( keypoints_dir=os.path.join(args.data_dir, f"{args.dataset}_val_keypoints"), use_augs=False, label_map=label_map, mode="val", max_frame_len=169, ) train_dataloader = data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True, ) val_dataloader = data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True, ) n_classes = 50 if args.dataset == "include": n_classes = 263 if args.model == "lstm": config = LstmConfig() if args.use_cnn: config.input_size = CnnConfig.output_dim model = LSTM(config=config, n_classes=n_classes) else: config = TransformerConfig(size=args.transformer_size) if args.use_cnn: config.input_size = CnnConfig.output_dim model = Transformer(config=config, n_classes=n_classes) model = model.to(device) optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=0.01) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.2) if args.use_pretrained == "resume_training": model, optimizer, scheduler = load_pretrained(args, n_classes, model, optimizer, scheduler) model_path = os.path.join(args.save_path, exp_name) + ".pth" es = EarlyStopping(patience=15, mode="max") for epoch in range(args.epochs): print(f"Epoch: {epoch+1}/{args.epochs}") train_loss, train_acc = train(train_dataloader, model, optimizer, device) val_loss, val_acc = validate(val_dataloader, model, device) logging.info( "Epoch: {}, train loss: {}, train acc: {}, val loss: {}, val acc: {}" .format(epoch + 1, train_loss, train_acc, val_loss, val_acc)) scheduler.step(val_acc) es( model_path=model_path, epoch_score=val_acc, model=model, optimizer=optimizer, scheduler=scheduler, ) if es.early_stop: print("Early stopping") break print("### Training Complete ###")
if multi_gpu: device = torch.device("cuda") if args.gpu == -1 else torch.device( f'cuda:{args.gpu}') assert args.train_batch_size % n_gpu == 0, f"Train batch size will need to be allocated equally across {n_gpu} gpus, but {args.train_batch_size} cannot be" assert args.test_batch_size % n_gpu == 0, f"Eval batch size will need to be allocated equally across {n_gpu} gpus, but {args.test_batch_size} cannot be" else: device = torch.device(f"cuda:{args.gpu}") torch.cuda.set_device(device) args.device = device np.random.seed(args.seed) torch.random.manual_seed(args.seed) if multi_gpu: torch.cuda.manual_seed_all(args.seed) # make Report object, stats_dict, and paths args.experiment_name = experiment_name = utils.get_experiment_name( args) if args.experiment_name is None else args.experiment_name if args.small_data and args.do_train: experiment_name += f"_DEBUG" model_path = os.path.join(args.save_dir, f'{experiment_name}.pth') retriever_path = os.path.join(args.save_dir, f'{experiment_name}_retriever.pth') print(f"\nStarting experiment: {experiment_name}") # make pretrained_model_path pretrained_model_path = os.path.join( args.save_dir, f'{args.pretrained_model}.pth') if args.pretrained_model else None pretrained_retriever_path = os.path.join( args.save_dir, f'{args.pretrained_retriever}.pth' ) if args.pretrained_retriever else None # report + stats report_name = f"report_{experiment_name}.txt"
self.log("loss", loss) self.log("acc", acc) return loss def validation_step(self, batch, batch_idx): img, label = batch out = self(img) loss = self.criterion(out, label) acc = self.accuracy(out, label) self.log("val_loss", loss) self.log("val_acc", acc) return loss if __name__ == "__main__": experiment_name = get_experiment_name(args) print(experiment_name) logger = pl.loggers.CometLogger( api_key=args.api_key, save_dir="log", project_name="image_classification_pytorch", experiment_name=experiment_name) args.api_key = None # Initialize API Key for privacy. net = Net(args) trainer = pl.Trainer(precision=args.precision, fast_dev_run=args.dry_run, gpus=args.gpus, benchmark=args.benchmark, logger=logger, max_epochs=args.max_epochs, weights_summary="full",
def train(model, dataset, data_augmentation, epochs, batch_size, beta, M, initial_lr, lr_schedule, strategy, output_dir, class_loss, cov_type): model_conf = model train_set, test_set, small_set = datasets.get_dataset(dataset) TRAIN_BUF, TEST_BUF = datasets.dataset_size[dataset] if data_augmentation: base_dataset = dataset.split("-")[0] print(f"Using image generator params from {base_dataset}") with open(f"./datasets/image-generator-config/{base_dataset}.yml", "r") as fh: params = yaml.safe_load(fh) print(params) train_dataset = tf.keras.preprocessing.image.ImageDataGenerator( **params) train_dataset.fit(train_set[0]) else: train_dataset = tf.data.Dataset.from_tensor_slices(train_set) \ .shuffle(TRAIN_BUF).batch(batch_size) test_dataset = tf.data.Dataset.from_tensor_slices(test_set) \ .shuffle(TEST_BUF).batch(batch_size) print( f"Training with {model} on {dataset} for {epochs} epochs (lr={initial_lr}, schedule={lr_schedule})" ) print( f"Params: batch-size={batch_size} beta={beta} M={M} lr={initial_lr} strategy={strategy}" ) optimizers, strategy_name, opt_params = losses.get_optimizer( strategy, lr, lr_schedule, dataset, batch_size) network_name, architecture = model.split("/") experiment_name = utils.get_experiment_name( f"{network_name}-{class_loss}-{cov_type}-{dataset}") print(f"Experiment name: {experiment_name}") artifact_dir = f"{output_dir}/{experiment_name}" print(f"Artifact directory: {artifact_dir}") train_log_dir = f"{artifact_dir}/logs/train" test_log_dir = f"{artifact_dir}/logs/test" train_summary_writer = tf.summary.create_file_writer(train_log_dir) test_summary_writer = tf.summary.create_file_writer(test_log_dir) # Instantiate model architecture = utils.parse_arch(architecture) model = nets.get_network(network_name)(architecture, datasets.input_dims[dataset], datasets.num_classes[dataset], cov_type, beta=beta, M=M) model.build(input_shape=(batch_size, *datasets.input_dims[dataset])) model.summary() print(f"Class loss: {class_loss}") model.class_loss = getattr(losses, f"compute_{class_loss}_class_loss") lr_labels = list(map(lambda x: f"lr_{x}", range(len(optimizers)))) train_step = train_algo2 if strategy.split( "/")[0] == "algo2" else train_algo1 print("Using trainstep: ", train_step) train_start_time = time.time() steps_per_epoch = int(np.ceil(train_set[0].shape[0] / batch_size)) for epoch in range(1, epochs + 1): start_time = time.time() print(f"Epoch {epoch}") m, am = train_step( model, optimizers, train_dataset.flow( train_set[0], train_set[1], batch_size=batch_size) if data_augmentation else train_dataset, train_summary_writer, M, lr_labels, strategy_name, opt_params, epoch, steps_per_epoch) m = m.result().numpy() am = am.result().numpy() print(utils.format_metrics("Train", m, am)) tfutils.log_metrics(train_summary_writer, metric_labels, m, epoch) tfutils.log_metrics(train_summary_writer, acc_labels, am, epoch) tfutils.log_metrics( train_summary_writer, lr_labels, map(lambda opt: opt._decayed_lr(tf.float32), optimizers), epoch) train_metrics = m.astype(float).tolist() + am.astype(float).tolist() end_time = time.time() test_metrics = evaluate(model, test_dataset, test_summary_writer, M, epoch) print(f"--- Time elapse for current epoch {end_time - start_time}") train_end_time = time.time() elapsed_time = (train_end_time - train_start_time) / 60. test_metrics_dict = dict(zip(metric_labels + acc_labels, test_metrics)) summary = dict( dataset=dataset, model=model_conf, strategy=strategy, beta=beta, epoch=epoch, M=M, lr=initial_lr, lr_schedule=lr_schedule, metrics=dict( train=dict(zip(metric_labels + acc_labels, train_metrics)), test=test_metrics_dict, ), class_loss=class_loss, cov_type=cov_type, batch_size=batch_size, elapsed_time=elapsed_time, # in minutes test_accuracy_L12=test_metrics_dict["accuracy_L12"], data_augmentation=data_augmentation) if model.latent_dim == 2: plot_helper.plot_2d_representation( model, small_set, title="Epoch=%d Strategy=%s Beta=%f M=%f" % (epoch, strategy, beta, M), path=f"{artifact_dir}/latent-representation.png") with train_summary_writer.as_default(): tf.summary.text("setting", json.dumps(summary, sort_keys=True, indent=4), step=0) with open(f"{artifact_dir}/summary.yml", 'w') as f: print(summary) yaml.dump(summary, f, default_flow_style=False) model.save_weights(f"{artifact_dir}/model") print(f"Training took {elapsed_time:.4f} minutes") print(f"Please see artifact at: {artifact_dir}")
from baseline_train import train as train_baseline from adaptive_train import train as train_adaptive from config import * from utils import get_experiment_name if __name__ == "__main__": if RANK and not PAIRWISE: raise Exception("Invalid Configuration!! Aborting") experiment_name = get_experiment_name(EXPERIMENT_NAME, PAIRWISE, RANK, ADAPTIVE, LOSS) if ADAPTIVE: pretrained_model_name = None # pretrained_model_name = 'baseline_L1' train_adaptive(experiment_name, pretrained_model_name=pretrained_model_name) else: train_baseline(experiment_name)