def seeded_exp(cmd): """Run seeded exps""" args = get_args(cmd) args.name = "{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}".format( args.sampling_strategy, args.diversify, args.shift_correction, "IW" if args.train_iw else "NOIW", "ITIW" if args.iterative_iw else "NOITIW", "reweight" if args.reweight else "", "norllsinfer" if not args.rlls_infer else "", "onlyrllsinfer" if args.only_rlls_infer else "", args.version, args.shift_strategy, args.dirichlet_alpha, args.dataset, args.warmstart_ratio, ) name = args.name + " " + str(args.seed) # Initialize comet.ml comet_api = api.API(api_key=comet_ml_key) # exps = comet_api.get_experiments( # "qasimwani", # project_name="active-label-shift-adaptation", # pattern=name) # for exp in exps: # print(exp) # if exp.get_name() == name: # raise ValueError("EXP EXISTS!") logger = Experiment(comet_ml_key, project_name="active-label-shift-adaptation") logger.set_name(name) logger.log_parameters(vars(args)) # Run experiments this_metrics = experiment(args, logger, name, seed=args.seed) print(this_metrics)
def experiment(): """Baseline exp""" args = get_args(None) logger = Experiment(comet_ml_key, project_name="active-label-shift-adaptation") logger.set_name("Baseline lr {} g {} bs {} {} {}".format( args.lr, args.gamma, args.batch_size, "simple " if args.simple_model else "", args.dataset)) logger.log_parameters(vars(args)) # Seed the experiment seed = args.seed torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) print("Running seed ", seed) torch.cuda.set_device(args.device) assert torch.cuda.is_available() # Shuffle dataset dataset = get_datasets(args) # Train h0 net_cls = get_net_cls(args) network = net_cls(args.num_cls).to(args.device) def log_fn(epoch): network.eval() accuracy = evaluate( network, dataset.iterate(args.infer_batch_size, False, split="test"), args.device, args) logger.log_metrics(accuracy, step=epoch) train(network, dataset=dataset, epochs=args.initial_epochs, args=args, log_fn=log_fn)
def experiment(): """Run finetuning label shift exp""" args = get_args(None) name = "Finetune {}:{}:{} {}:{} {}:{}:{}:{} {}{}v{}".format( args.dataset, args.dataset_cap, args.warmstart_ratio, args.shift_strategy, args.dirichlet_alpha, args.shift_correction, args.rlls_reg, args.rlls_lambda, args.lr, "IW " if args.train_iw else "NOIW ", "ITIW " if args.iterative_iw else "NOITIW ", args.version, ) # Initialize comet.ml if args.log: comet_api = api.API(api_key=comet_ml_key) exps = comet_api.get_experiments( "ericzhao28", project_name="active-label-shift-adaptation", pattern=name) for exp in exps: if exp.get_name() == name: raise ValueError("EXP EXISTS!") logger = Experiment(comet_ml_key, project_name="active-label-shift-adaptation") logger.set_name(name) logger.log_parameters(vars(args)) # Seed the experiment seed = args.seed torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) print("Running seed ", seed) torch.cuda.set_device(args.device) assert torch.cuda.is_available() # Shuffle dataset dataset = get_datasets(args) dataset.label_ptrs(np.arange(dataset.online_len())) # Train h0 net_cls = get_net_cls(args) network = net_cls(args.num_cls).to(args.device) def log_fn(epoch): network.eval() accuracy = evaluate(network, dataset.iterate(args.infer_batch_size, False, split="test"), args.device, args, label_weights=dataset.label_weights) print(accuracy) logger.log_metrics(accuracy, prefix="initial", step=epoch) train(network, dataset=dataset, epochs=args.initial_epochs, args=args, log_fn=log_fn) # Get source shift corrections lsmse = label_shift(network, dataset, args) def log_fn_shifted(epoch): network.eval() if args.iterative_iw: label_shift(network, dataset, args) accuracy = evaluate(network, dataset.iterate(args.infer_batch_size, False, split="test"), args.device, args, label_weights=dataset.label_weights) print(accuracy) logger.log_metrics(accuracy, prefix="shifted", step=epoch) train(network, dataset=dataset, epochs=args.initial_epochs, args=args, log_fn=log_fn_shifted) if args.iterative_iw: lsmse = label_shift(network, dataset, args) logger.log_metrics({"IW MSE": lsmse}, prefix="initial")
names = {} with open(os.path.join(dataset_path, 'classes.txt')) as f: for line in f: pieces = line.strip().split() class_id = pieces[0] names[class_id] = ' '.join(pieces[1:]) return names def load_hierarchy(dataset_path=''): """Load label hierarchy.""" parents = {} with open(os.path.join(dataset_path, 'hierarchy.txt')) as f: for line in f: pieces = line.strip().split() child_id, parent_id = pieces parents[child_id] = parent_id return parents if __name__ == '__main__': train_dataset = NABirds('/content/data', train=True, download=False) args = get_args("--nabirdstype child") print(train_dataset.change_class_map(args)) print(train_dataset.get_target(0)) print(len(np.unique(list(train_dataset.label_map.values()))))