def seeded_exp(cmd):
"""Run seeded exps"""
args = get_args(cmd)
args.name = "{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}zz{}".format(
args.sampling_strategy,
args.diversify,
args.shift_correction,
"IW" if args.train_iw else "NOIW",
"ITIW" if args.iterative_iw else "NOITIW",
"reweight" if args.reweight else "",
"norllsinfer" if not args.rlls_infer else "",
"onlyrllsinfer" if args.only_rlls_infer else "",
args.version,
args.shift_strategy,
args.dirichlet_alpha,
args.dataset,
args.warmstart_ratio,
)
name = args.name + " " + str(args.seed)
# Initialize comet.ml
comet_api = api.API(api_key=comet_ml_key)
# exps = comet_api.get_experiments(
# "qasimwani",
# project_name="active-label-shift-adaptation",
# pattern=name)
# for exp in exps:
# print(exp)
# if exp.get_name() == name:
# raise ValueError("EXP EXISTS!")
logger = Experiment(comet_ml_key,
project_name="active-label-shift-adaptation")
logger.set_name(name)
logger.log_parameters(vars(args))
# Run experiments
this_metrics = experiment(args, logger, name, seed=args.seed)
print(this_metrics)
def experiment():
"""Baseline exp"""
args = get_args(None)
logger = Experiment(comet_ml_key,
project_name="active-label-shift-adaptation")
logger.set_name("Baseline lr {} g {} bs {} {} {}".format(
args.lr, args.gamma, args.batch_size,
"simple " if args.simple_model else "", args.dataset))
logger.log_parameters(vars(args))
# Seed the experiment
seed = args.seed
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
print("Running seed ", seed)
torch.cuda.set_device(args.device)
assert torch.cuda.is_available()
# Shuffle dataset
dataset = get_datasets(args)
# Train h0
net_cls = get_net_cls(args)
network = net_cls(args.num_cls).to(args.device)
def log_fn(epoch):
network.eval()
accuracy = evaluate(
network, dataset.iterate(args.infer_batch_size,
False,
split="test"), args.device, args)
logger.log_metrics(accuracy, step=epoch)
train(network,
dataset=dataset,
epochs=args.initial_epochs,
args=args,
log_fn=log_fn)
def experiment():
"""Run finetuning label shift exp"""
args = get_args(None)
name = "Finetune {}:{}:{} {}:{} {}:{}:{}:{} {}{}v{}".format(
args.dataset,
args.dataset_cap,
args.warmstart_ratio,
args.shift_strategy,
args.dirichlet_alpha,
args.shift_correction,
args.rlls_reg,
args.rlls_lambda,
args.lr,
"IW " if args.train_iw else "NOIW ",
"ITIW " if args.iterative_iw else "NOITIW ",
args.version,
)
# Initialize comet.ml
if args.log:
comet_api = api.API(api_key=comet_ml_key)
exps = comet_api.get_experiments(
"ericzhao28",
project_name="active-label-shift-adaptation",
pattern=name)
for exp in exps:
if exp.get_name() == name:
raise ValueError("EXP EXISTS!")
logger = Experiment(comet_ml_key,
project_name="active-label-shift-adaptation")
logger.set_name(name)
logger.log_parameters(vars(args))
# Seed the experiment
seed = args.seed
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
print("Running seed ", seed)
torch.cuda.set_device(args.device)
assert torch.cuda.is_available()
# Shuffle dataset
dataset = get_datasets(args)
dataset.label_ptrs(np.arange(dataset.online_len()))
# Train h0
net_cls = get_net_cls(args)
network = net_cls(args.num_cls).to(args.device)
def log_fn(epoch):
network.eval()
accuracy = evaluate(network,
dataset.iterate(args.infer_batch_size,
False,
split="test"),
args.device,
args,
label_weights=dataset.label_weights)
print(accuracy)
logger.log_metrics(accuracy, prefix="initial", step=epoch)
train(network,
dataset=dataset,
epochs=args.initial_epochs,
args=args,
log_fn=log_fn)
# Get source shift corrections
lsmse = label_shift(network, dataset, args)
def log_fn_shifted(epoch):
network.eval()
if args.iterative_iw:
label_shift(network, dataset, args)
accuracy = evaluate(network,
dataset.iterate(args.infer_batch_size,
False,
split="test"),
args.device,
args,
label_weights=dataset.label_weights)
print(accuracy)
logger.log_metrics(accuracy, prefix="shifted", step=epoch)
train(network,
dataset=dataset,
epochs=args.initial_epochs,
args=args,
log_fn=log_fn_shifted)
if args.iterative_iw:
lsmse = label_shift(network, dataset, args)
logger.log_metrics({"IW MSE": lsmse}, prefix="initial")
names = {}
with open(os.path.join(dataset_path, 'classes.txt')) as f:
for line in f:
pieces = line.strip().split()
class_id = pieces[0]
names[class_id] = ' '.join(pieces[1:])
return names
def load_hierarchy(dataset_path=''):
"""Load label hierarchy."""
parents = {}
with open(os.path.join(dataset_path, 'hierarchy.txt')) as f:
for line in f:
pieces = line.strip().split()
child_id, parent_id = pieces
parents[child_id] = parent_id
return parents
if __name__ == '__main__':
train_dataset = NABirds('/content/data', train=True, download=False)
args = get_args("--nabirdstype child")
print(train_dataset.change_class_map(args))
print(train_dataset.get_target(0))
print(len(np.unique(list(train_dataset.label_map.values()))))