def main(): parser = argparse.ArgumentParser() add_common_cli_args(parser) cli_args = parser.parse_args() opts = get_params( simclr_sender=cli_args.simclr_sender, shared_vision=cli_args.shared_vision, loss_type=cli_args.loss_type, discrete_evaluation_simclr=cli_args.discrete_evaluation_simclr, ) if cli_args.pdb: breakpoint() print(f"| Loading model from {cli_args.checkpoint_path} ...") game = get_game(opts, cli_args.checkpoint_path) print("| Model loaded") dataloader = get_random_noise_dataloader( use_augmentations=cli_args.evaluate_with_augmentations) print("| Starting evaluation ...") loss, soft_acc, game_acc, _ = evaluate(game=game, data=dataloader) print( f"| Loss: {loss}, soft_accuracy (out of 100): {soft_acc * 100}, game_accuracy (out of 100): {game_acc * 100}" )
def main(): parser = argparse.ArgumentParser() add_common_cli_args(parser) cli_args = parser.parse_args() opts = get_params( simclr_sender=cli_args.simclr_sender, shared_vision=cli_args.shared_vision, loss_type=cli_args.loss_type, discrete_evaluation_simclr=cli_args.discrete_evaluation_simclr, ) if cli_args.pdb: breakpoint() print(f"| Loading model from {cli_args.checkpoint_path} ...") game = get_game(opts, cli_args.checkpoint_path) print("| Model loaded.") print(f"| Fetching data from {cli_args.test_dataset_dir}...") dataloader = get_dataloader( dataset_dir=cli_args.test_dataset_dir, use_augmentations=cli_args.evaluate_with_augmentations, ) print("| Test data fetched.") print("| Starting evaluation ...") loss, soft_acc, game_acc, full_interaction = evaluate(game=game, data=dataloader) print( f"| Loss: {loss}, soft_accuracy (out of 100): {soft_acc * 100}, game_accuracy (out of 100): {game_acc * 100}" ) if cli_args.dump_interaction_folder: save_interaction(interaction=full_interaction, log_dir=cli_args.dump_interaction_folder) print(f"| Interaction saved at {cli_args.dump_interaction_folder}") print("Finished evaluation.")
def main(): parser = argparse.ArgumentParser() parser.add_argument("--num_clusters", type=int, default=1000) parser.add_argument("--train_dataset_dir", required=True) add_common_cli_args(parser) cli_args = parser.parse_args() opts = get_params( simclr_sender=cli_args.simclr_sender, shared_vision=cli_args.shared_vision, loss_type=cli_args.loss_type, discrete_evaluation_simclr=cli_args.discrete_evaluation_simclr ) if cli_args.pdb: breakpoint() print(f"| Fetching train data from {cli_args.train_dataset_dir} to learn clusters...") train_dataloader = get_dataloader( dataset_dir=cli_args.train_dataset_dir, use_augmentations=cli_args.evaluate_with_augmentations, ) print("| Fetched train data.") print(f"| Fetching test data from {cli_args.test_dataset_dir}...") test_dataloader = get_dataloader( dataset_dir=cli_args.test_dataset_dir, use_augmentations=cli_args.evaluate_with_augmentations, ) print("| Fetched test data") print(f"| Loading model from {cli_args.checkpoint_path} ...") game = get_game(opts, cli_args.checkpoint_path) print("| Model loaded.") print("| Starting evaluation ...") _, _, _, interaction = evaluate( game=game, data=train_dataloader ) print("| Finished processing train_data") print("| Clustering resnet outputs ...") k_means_clusters = assign_kmeans_labels(interaction, cli_args.num_clusters) print("| Done clustering resnet outputs") print("| Running evaluation on the test set ...") loss, soft_acc, game_acc, interaction = evaluate_test_set( game=game, data=test_dataloader, k_means_clusters=k_means_clusters, num_clusters=cli_args.num_clusters ) print("| Done evaluation on the test set") print(f"| Loss: {loss}, soft_accuracy (out of 100): {soft_acc * 100}, game_accuracy (out of 100): {game_acc * 100}") if cli_args.dump_interaction_folder: print("| Saving interaction ...") save_interaction( interaction=interaction, log_dir=cli_args.dump_interaction_folder ) print(f"| Interaction saved at {cli_args.dump_interaction_folder}") print("Finished evaluation.")