def save_features(model, losses_bag, data_loader, features_dir, params):
penultimate_dict, features_dict = evaluate(data_loader,
model,
losses_bag,
params,
save_latent=True)
save_pickle(os.path.join(features_dir, 'novel_penultimate.plk'),
penultimate_dict)
save_pickle(os.path.join(features_dir, 'novel_features.plk'),
features_dict)
if __name__ == '__main__':
params = parse_args('graph')
random.seed(914637)
image_size = 84
save_dir = './weights'
data_dir = {
'cifar': './filelists/cifar/',
'CUB': './filelists/CUB/',
'miniImagenet': './filelists/miniImagenet/'
}
novel_file = data_dir[params.dataset] + 'novel.json'
params.checkpoint_dir = '%s/checkpoints/%s/%s/%s' % (
save_dir, params.dataset, params.model, params.run_name)
n_way, n_shot, n_val = params.n_way, params.n_shot, params.n_val
novel_datamgr = SetDataManager(novel_file, image_size, n_way, n_shot,
if all_feats is None:
all_feats = f.create_dataset('all_feats',
[max_count] + list(feats.size()[1:]),
dtype='f')
all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy()
all_labels[count:count + feats.size(0)] = y.cpu().numpy()
count = count + feats.size(0)
count_var = f.create_dataset('count', (1, ), dtype='i')
count_var[0] = count
f.close()
if __name__ == '__main__':
params = parse_args('save_features')
assert params.method != 'maml' and params.method != 'maml_approx', 'maml do not support save_feature and run'
if 'Conv' in params.model:
if params.dataset in ['omniglot', 'cross_char']:
image_size = 28
else:
image_size = 84
else:
image_size = 224
if params.dataset in ['omniglot', 'cross_char']:
assert params.model == 'Conv4' and not params.train_aug, 'omniglot only support Conv4 without augmentation'
params.model = 'Conv4S'
split = params.split
model.eval()
correct, total = 0, 0
for i, (x, y) in enumerate(testloader):
x, y = x.cuda(), y.cuda()
# print(y)
x = pgd.attack(model, x, y)
scores, _ = model.forward(x)
predicted = torch.argmax(scores, 1)
correct += (predicted == y).sum().item()
total += predicted.size(0)
print('Accuracy {:f}'.format(100. * correct / total))
if __name__ == '__main__':
np.random.seed(10)
params = parse_args()
print(params)
if params.dataset == 'cifar10':
params.num_classes = 10
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
acc_all_ori = np.asarray(acc_all_ori)
acc_mean_ori = np.mean(acc_all_ori)
acc_std_ori = np.std(acc_all_ori)
print('BSR+ENT: %d Test Acc = %4.2f%% +- %4.2f%%' %
(iter_num, acc_mean_ori, 1.96 * acc_std_ori / np.sqrt(iter_num)))
acc_all_lp = np.asarray(acc_all_lp)
acc_mean_lp = np.mean(acc_all_lp)
acc_std_lp = np.std(acc_all_lp)
print('BSR+LP+ENT: %d Test Acc = %4.2f%% +- %4.2f%%' %
(iter_num, acc_mean_lp, 1.96 * acc_std_lp / np.sqrt(iter_num)))
if __name__ == '__main__':
np.random.seed(10)
params = parse_args('finetune')
image_size = 224
iter_num = 600
params.method = 'bsr'
few_shot_params = dict(n_way=params.test_n_way,
n_support=params.n_shot,
n_query=15)
freeze_backbone = params.freeze_backbone
if params.dtarget == 'ISIC':
print("Loading ISIC")
datamgr = ISIC_few_shot.SetDataManager(image_size,
n_eposide=iter_num,
**few_shot_params)
def exp_grid(self, choose_by='val_acc_mean', mode='from_scratch'):
'''
Args:
mode (str): 'from_scratch'|'resume'|'draw_tasks'|'tmp_pkl' # 'draw_tasks' could be deprecated???
'''
print('exp_grid() start.')
print('self.base_params:', self.base_params)
base_params = self.base_params.copy() # set to test_n_way = default(5)
if mode == 'tmp_pkl':
if 'test_n_way' in base_params:
print('change base_params["test_n_way"] to be 5.')
base_params['test_n_way'] = 5
default_args = {} # the raw default args of the code
default_args['train'] = parse_args('train', parse_str='')
default_args['test'] = parse_args('test', parse_str='')
possible_params = self.possible_params
if mode == 'tmp_pkl':
for key in possible_params['general']:
if len(possible_params['general'][key]) != 1:
raise ValueError(
'general param length should be 1 for mode: tmp_pkl')
for key in possible_params['test']:
if len(possible_params['test'][key]) != 1:
raise ValueError(
'test param length should be 1 for mode: tmp_pkl')
train_args = get_modified_args(default_args['train'], {
**base_params,
**self.fixed_params['train']
})
test_args = get_modified_args(default_args['test'], {
**self.base_params,
**self.fixed_params['test']
}) # test_n_way should be 2
all_general_possible_params = get_all_params_comb(
possible_params=possible_params['general'])
all_test_possible_params = get_all_params_comb(
possible_params=possible_params['test'])
csv_path = os.path.join(self.record_folder,
self.fixed_params['test']['csv_name'])
# TODO: refactor to many functions with argument: mode
is_csv_exists = os.path.exists(csv_path)
########## deal with initial csv ##########
if mode == 'from_scratch':
if not is_csv_exists:
# create empty csv
print("Creating file: '%s' since not exist" % (csv_path))
with open(csv_path, "w") as file:
empty_df = pd.DataFrame(columns=['dataset'])
empty_df.to_csv(file, header=True, index=False)
pkl_postfix_str = '_' + self.pkl_postfix #+ '.pkl'
pkl_postfix_str += '.pkl'
pkl_path = csv_path.replace(
'.csv', pkl_postfix_str) # restore ALL experiments in this class
is_pkl_exists = os.path.exists(pkl_path)
########## load pickle data ##########
if mode == 'resume':
print('loading self.results_pkl from:', pkl_path)
if is_pkl_exists:
with open(pkl_path, 'rb') as handle:
self.results_pkl = pickle.load(handle)
else:
logging.warning(
'No previous result pickle file: %s, only record self.results_pkl from scratch.'
% (pkl_path))
print('self.results_pkl at begin, len =', len(self.results_pkl))
elif mode == 'tmp_pkl':
if is_pkl_exists:
print('pickle file: %s, already exists.' % (pkl_path))
return
for general_params in all_general_possible_params:
########## decide if should_train ##########
if mode in ['resume', 'from_scratch']:
# if mode == 'resume':# or mode == 'draw_tasks':
print()
print('=' * 20, 'Checking if already trained in:', csv_path,
'=' * 20)
print('base_params:', base_params)
print('general_params:', general_params)
# read csv
loaded_df = pd.read_csv(csv_path)
# check if ALL test_params has been experimented. if so, then do NOT even train model
check_df = loaded_df.copy()
check_param = {**base_params, **general_params}
# include default columns if param not in dataframe
default_train_args = parse_args('train', parse_str='')
default_train_param = vars(default_train_args)
for param in check_param:
if param not in check_df.columns:
print('param:', param)
if default_train_param[param] is None:
check_df[param] = np.nan
else:
check_df[param] = default_train_param[param]
check_df = get_matched_df(check_param, check_df)
num_experiments = len(check_df)
should_train = num_experiments == 0
# elif mode == 'from_scratch':
# should_train = True
elif mode in ['draw_tasks', 'tmp_pkl']:
should_train = False
########## training ##########
if should_train:
# release cuda memory
torch.cuda.empty_cache()
# train model
print()
print('=' * 20, 'Training', '=' * 20)
print('general_params:', general_params)
modified_train_args = get_modified_args(
train_args, general_params)
if mode == 'resume':
modified_train_args.resume = True
args_sanity_check(modified_train_args, script='train')
train_result = exp_train_val(modified_train_args)
else:
if mode == 'draw_tasks':
print('NO need to train when draw_tasks.')
elif mode == 'resume':
print('NO need to train since already trained in record:',
csv_path)
print('Summing up current record...')
choose_by = 'val_acc_mean'
top_k = None
self.sum_up_results(choose_by=choose_by, top_k=top_k)
else:
print('NO need to train since already trained in record:',
csv_path)
modified_test_args = get_modified_args(test_args, general_params)
# loop over testing settings under each general setting
for test_params in all_test_possible_params:
########## check if should test ##########
if mode == 'resume':
print('\n', '=' * 20,
'Checking if already did experiments', '=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
check_df = loaded_df.copy()
constrained_param = {
**self.base_params,
**general_params,
**test_params
}
# add default params, delete useless params & dependent variables (result)
default_args = parse_args('test', parse_str='')
default_param = vars(default_args)
check_param = {**default_param, **constrained_param}
# delete vars not for checking
del_vars = self.negligible_vars + self.dependent_vars
for var in del_vars:
if var in check_param:
del check_param[var]
# include default columns if param not in dataframe
for param in check_param:
if param not in check_df.columns:
print('param:', param)
if default_param[param] is None:
check_df[param] = np.nan
else:
check_df[param] = default_param[param]
# print(check_df)
# yooooo
check_df = get_matched_df(
check_param,
check_df) # should loaded_df be check_df ???
num_test_experiments = len(check_df)
if num_test_experiments > 0: # already experiments
print(
'NO need to test since already tested %s times in record: %s'
% (num_test_experiments, csv_path))
print(check_df)
continue
else:
print(
'Need to do testing since there\'s NO experiment in data.'
)
elif mode == 'tmp_pkl':
pass
final_test_args = get_modified_args(modified_test_args,
test_params)
write_record = {**general_params, **test_params}
########## write train_acc to record dict ##########
if mode in ['resume', 'from_scratch']:
# if mode == 'resume': #in ['resume', 'draw_tasks']:
source_val = True
# write source_val_acc
if source_val and 'cross' in base_params[
'dataset'] and base_params[
'dataset'] != 'cross': # in ['cross_base20cl', 'cross_char', 'cross_char_half', 'cross_char_quarter', 'cross_char2', 'cross_char2_base1lang']:
if should_train:
write_record['source_val_acc'] = train_result[
'source_val_acc']
else:
# to get source_val_acc, so no need test_params
check_df = loaded_df.copy()
check_df = get_matched_df(
{
**self.base_params,
**general_params
}, check_df)
write_record['source_val_acc'] = check_df[
'source_val_acc'].iloc[0]
else:
write_record['source_val_acc'] = None
# write train_acc
if should_train:
write_record['train_acc_mean'] = train_result[
'train_acc']
else:
# to get train_acc, so no need test_params
check_df = loaded_df.copy()
check_df = get_matched_df(
{
**self.base_params,
**general_params
}, check_df)
write_record['train_acc_mean'] = check_df[
'train_acc_mean'].iloc[0]
########## save_features & test ##########
if mode in ['from_scratch', 'resume', 'tmp_pkl']:
########### judge if should do several exp for frac_ensemble ###########
tmp = final_test_args # to make variable name shorter
is_str_single_frac = isinstance(
tmp.frac_ensemble,
str) and ',' not in tmp.frac_ensemble
is_float_none_frac = isinstance(
tmp.frac_ensemble, float
) or tmp.frac_ensemble == 1 or tmp.frac_ensemble is None
is_single_exp = is_float_none_frac or is_str_single_frac
if is_str_single_frac:
final_test_args.frac_ensemble = frac_ensemble_str2var(
params.frac_ensemble)
if is_single_exp: # common frac_ensemble
splits = ['val', 'novel'] # temporary no 'train'
for split in splits: # val, novel
##### get args #####
split_final_test_args = copy_args(final_test_args)
split_final_test_args.split = split
print('\n', '=' * 20, 'Saving Features', '=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
print('data split:', split)
args_sanity_check(split_final_test_args,
script='save_features')
args_sanity_check(split_final_test_args,
script='test')
########## save features ##########
exp_save_features(copy_args(split_final_test_args))
print('\n', '=' * 20, 'Testing', '=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
print('data split:', split)
n_episodes = 10 if split_final_test_args.debug or mode == 'draw_tasks' else 600
########## testing and record to dict ##########
exp_record, task_datas = exp_test(
copy_args(split_final_test_args),
n_episodes=n_episodes,
should_del_features=True
) #, show_data=show_data)
write_record['epoch'] = exp_record['epoch']
write_record[split +
'_acc_mean'] = exp_record['acc_mean']
write_record[split +
'_acc_std'] = exp_record['acc_std']
torch.cuda.empty_cache()
########## record to csv ##########
if mode in ['from_scratch', 'resume']:
print('Saving record to:', csv_path)
record_to_csv(final_test_args,
write_record,
csv_path=csv_path)
print('=' * 20, 'Current Experiments', '=' * 20)
choose_by = 'val_acc_mean'
top_k = None
self.sum_up_results(choose_by, top_k)
########## record to pickle ##########
write_record[
'novel_task_datas'] = task_datas # currently ignore val_task_datas
self.results_pkl.append(write_record)
print('Saving self.results_pkl into:', pkl_path)
with open(pkl_path, 'wb') as handle:
pickle.dump(self.results_pkl,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
else: ########## multiple frac_ensemble ##########
# make frac_ensemble list
frac_ls = final_test_args.frac_ensemble.split(',')
frac_ls = list(map(frac_ensemble_str2var, frac_ls))
print('frac_ls:', frac_ls)
########## check if each frac_ensemble already done in exps ##########
if mode == 'resume':
for frac in frac_ls.copy():
print(
'\n', '=' * 20,
'Checking if already did "frac_ensemble" experiments',
'=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
check_df = loaded_df.copy()
constrained_param = {
**self.base_params,
**general_params,
**test_params
}
# only add this line different from before
constrained_param['frac_ensemble'] = frac
# add default params, delete useless params & dependent variables (result)
default_args = parse_args('test', parse_str='')
default_param = vars(default_args)
check_param = {
**default_param,
**constrained_param
}
# delete vars not for checking
del_vars = self.negligible_vars + self.dependent_vars
for var in del_vars:
if var in check_param:
del check_param[var]
check_df = get_matched_df(
check_param, loaded_df)
num_test_experiments = len(check_df)
if num_test_experiments > 0: # already experiments
print(
'NO need to test since already tested %s times in record: %s'
% (num_test_experiments, csv_path))
print(check_df)
frac_ls.remove(frac)
else:
print(
'Need to do testing for frac_ensemble:',
frac,
'since there\'s NO experiment in data.'
)
########## do exps for multi-frac_ensemble ##########
if len(frac_ls) > 0:
final_test_args.frac_ensemble = frac_ls
##### initialize record for different frac #####
frac_write_records = [
copy.deepcopy(write_record)
for _ in range(len(frac_ls))
]
##### start exps #####
splits = ['val', 'novel']
for split in splits: # val, novel
##### get args #####
split_final_test_args = copy_args(
final_test_args)
split_final_test_args.split = split
print('\n', '=' * 20, 'Saving Features',
'=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
print('data split:', split)
##### sanity check #####
args_sanity_check(split_final_test_args,
script='save_features')
args_sanity_check(split_final_test_args,
script='test')
########## save features ##########
exp_save_features(
copy_args(split_final_test_args))
print('\n', '=' * 20, 'Testing', '=' * 20)
print('general_params:', general_params)
print('test_params:', test_params)
print('data split:', split)
n_episodes = 10 if split_final_test_args.debug or mode == 'draw_tasks' else 600
########## testing and record to dict ##########
##### return n_frac exps results #####
frac_exp_records, frac_task_datas = exp_test(
copy_args(split_final_test_args),
n_episodes=n_episodes,
should_del_features=True)
for frac_id, frac in enumerate(frac_ls):
frac_write_record = frac_write_records[
frac_id]
exp_record = frac_exp_records[frac_id]
frac_write_record['frac_ensemble'] = frac
frac_write_record['epoch'] = exp_record[
'epoch']
frac_write_record[
split +
'_acc_mean'] = exp_record['acc_mean']
frac_write_record[
split +
'_acc_std'] = exp_record['acc_std']
torch.cuda.empty_cache()
########## record n_frac exps to csv ##########
if mode in ['from_scratch', 'resume']:
print('Saving record to:', csv_path)
for frac_id, frac in enumerate(frac_ls):
frac_write_record = frac_write_records[
frac_id]
tmp_test_args = copy_args(final_test_args)
tmp_test_args.frac_ensemble = frac
record_to_csv(tmp_test_args,
frac_write_record,
csv_path=csv_path)
print('=' * 20, 'Current Experiments',
'=' * 20)
choose_by = 'val_acc_mean'
top_k = None
self.sum_up_results(choose_by, top_k)
########## record n_frac exps to pickle ##########
for frac_id, frac in enumerate(frac_ls):
frac_write_record = frac_write_records[frac_id]
frac_write_record[
'novel_task_datas'] = frac_task_datas[
frac_id] # currently ignore val_task_datas
self.results_pkl.append(frac_write_record)
print('Saving self.results_pkl into:', pkl_path)
with open(pkl_path, 'wb') as handle:
pickle.dump(self.results_pkl,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
else:
print(
'=' * 20,
'all frac_ensemble exps had been done before.',
'=' * 20)
torch.cuda.empty_cache()
########## sum up results ##########
# TODO: can also loop dataset
if mode in ['from_scratch', 'resume', 'draw_tasks']:
for choose_by in ['val_acc_mean', 'novel_acc_mean']:
# read csv to compare results
top_k = None
self.sum_up_results(choose_by, top_k)
if mode == 'draw_tasks':
print('loading self.results_pkl from:', pkl_path)
best_model_all_tasks = get_best_all_tasks(pkl_path=pkl_path)
print('sorting tasks...')
sorted_tasks = sorted(
best_model_all_tasks,
key=lambda i: float(i['acc'])) # in ascending order
print('Draw Worst Tasks...')
save_img_folder = os.path.join(self.record_folder, 'imgs')
draw_tasks(
sorted_tasks,
n_tasks=3,
save_img_folder=save_img_folder,
exp_postfix=self.pkl_postfix
) # utilize self.results_pkl, save all_tasks of best res
# 暂时弃用
from visualize_datapre import visualize_datapre
from io_utils import parse_args
import os, random
from my_utils import visualize_support_imgs
if __name__ == "__main__":
params = parse_args('mytest')
loadfile = os.path.join('filelists', params.test_dataset, 'novel.json')
sub_meta, class_names = visualize_datapre(params, loadfile)
random.seed(0)
# params
test_n_way = params.test_n_way
test_n_shot = params.test_n_shot
test_n_query = params.test_n_query
image_size = params.image_size
classes_id = sub_meta.keys()
selected_classes_id = random.sample(classes_id, test_n_way)
# random selected images_path
selected_imgs = {}
for i in selected_classes_id:
sub_imgs = sub_meta[i]
sub_selected_imgs = random.sample(sub_imgs, test_n_shot + test_n_query)
selected_imgs[i] = sub_selected_imgs
visualize_support_imgs(selected_imgs, class_names, test_n_shot, image_size)
-------
None
"""
exp = Experiment(args)
input_data, data_info = exp.get_input()
model, method_type, encoding_strategy = exp.get_model()
if not args['grid_search']:
param_grid = {}
if method_type != data_info["prediction_type"]:
raise RuntimeError("Experiment failed! "
"Dataset type is {} "
"but method type is {}".format(
data_info["prediction_type"], method_type))
generate_results(model=model,
input_data=input_data,
tmpdir=exp.tmpdir,
target_name=data_info['target_name'],
_id=args['_id'],
mode=method_type,
filename=data_info['filename'],
categories=data_info['categories'],
ordinals=data_info['ordinals'],
encoding_strategy=encoding_strategy,
param_grid=param_grid)
if __name__ == "__main__":
args, param_grid = parse_args()
main(args, param_grid)
if all_feats is None:
all_feats = f.create_dataset("all_feats",
[max_count] + list(feats.size()[1:]),
dtype="f")
all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy()
all_labels[count:count + feats.size(0)] = y.cpu().numpy()
count = count + feats.size(0)
count_var = f.create_dataset("count", (1, ), dtype="i")
count_var[0] = count
f.close()
if __name__ == "__main__":
params = parse_args("save_features")
assert (params.method != "maml" and params.method != "maml_approx"
), "maml do not support save_feature and run"
if "Conv" in params.model:
if params.dataset in ["omniglot", "cross_char"]:
image_size = 28
else:
image_size = 84
else:
image_size = 224
if params.dataset in ["omniglot", "cross_char"]:
assert (params.model == "Conv4" and not params.train_aug
), "omniglot only support Conv4 without augmentation"
params.model = "Conv4S"
acc = model.test_loop(val_loader)
if acc > max_acc: # for baseline and baseline++, we don't use validation here so we let acc = -1
print("--> Best model! save...")
max_acc = acc
outfile = os.path.join(params.checkpoint_dir, 'best_model.tar')
torch.save({'epoch': epoch, 'state': model.state_dict()}, outfile)
if (epoch % params.save_freq == 0) or (epoch == stop_epoch - 1):
outfile = os.path.join(params.checkpoint_dir, '{:d}.tar'.format(epoch))
torch.save({'epoch': epoch, 'state': model.state_dict()}, outfile)
return model
if __name__ == '__main__':
params = parse_args('train')
_set_seed(parse_args('train').seed)
if params.dataset == 'cross':
base_file = configs.data_dir['miniImagenet'] + 'all.json'
val_file = configs.data_dir['CUB'] + 'val.json'
elif params.dataset == 'cross_char':
base_file = configs.data_dir['omniglot'] + 'noLatin.json'
val_file = configs.data_dir['emnist'] + 'val.json'
else:
base_file = configs.data_dir[params.dataset] + 'base.json'
val_file = configs.data_dir[params.dataset] + 'val.json'
if 'Conv' in params.model:
if params.dataset in ['omniglot', 'cross_char']:
image_size = 28
else:
if all_feats is None:
all_feats = f.create_dataset('all_feats',
[max_count] + list(feats.size()[1:]),
dtype='f')
all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy()
all_labels[count:count + feats.size(0)] = y.cpu().numpy()
count = count + feats.size(0)
count_var = f.create_dataset('count', (1, ), dtype='i')
count_var[0] = count
f.close()
if __name__ == '__main__':
params = parse_args('eval')
assert params.method != 'maml' and params.method != 'maml_approx', 'maml do not support save_feature and run'
if 'Conv' in params.model:
if params.dataset in ['omniglot', 'cross_char']:
image_size = 28
else:
image_size = 84
else:
image_size = 224
split = params.split
loadfile = configs.data_dir[params.dataset] + split + '.json'
if params.checkpoint_dir == '':
checkpoint_dir = '%s/checkpoints/%s/%s_%s' % (
z_all = torch.from_numpy(np.array(z_all))
model.n_query = n_query
if adaptation:
scores = model.set_forward_adaptation(z_all, is_feature=True)
else:
scores = model.set_forward(z_all, is_feature=True)
pred = scores.data.cpu().numpy().argmax(axis=1)
y = np.repeat(range(n_way), n_query)
acc = np.mean(pred == y) * 100
return acc
if __name__ == '__main__':
args = parse_args('test')
if args.seed is not None:
torch.manual_seed(args.seed)
acc_all = []
iter_num = 600
few_shot_args = dict(n_way=args.test_n_way, n_support=args.n_shot)
l3_model = None
# Always load vocab for compatibility
if args.dataset == 'CUB':
vocab = lang_utils.load_vocab(configs.lang_dir)
json.dump(metrics,
fout,
sort_keys=True,
indent=4,
separators=(',', ': '))
# If didn't train, save model anyways
if stop_epoch == 0:
outfile = os.path.join(args.checkpoint_dir, 'best_model.tar')
torch.save({'epoch': stop_epoch, 'state': model.state_dict()}, outfile)
return model
if __name__ == '__main__':
args = parse_args('train')
if args.seed is not None:
torch.manual_seed(args.seed)
# I don't seed the np rng since dataset loading uses multiprocessing with random choices.
# https://github.com/numpy/numpy/issues/9650
# Unavoidable undeterminism here for now
if args.dataset == 'cross':
base_file = configs.data_dir['miniImagenet'] + 'all.json'
val_file = configs.data_dir['CUB'] + 'val.json'
elif args.dataset == 'cross_char':
base_file = configs.data_dir['omniglot'] + 'noLatin.json'
val_file = configs.data_dir['emnist'] + 'val.json'
else:
base_file = configs.data_dir[args.dataset] + 'base.json'
z_all = torch.from_numpy(np.array(z_all))
model.n_query = n_query
if adaptation:
scores = model.set_forward_adaptation(z_all, is_feature=True)
else:
scores = model.set_forward(z_all, is_feature=True)
pred = scores.data.cpu().numpy().argmax(axis=1)
y = np.repeat(range(n_way), n_query)
acc = np.mean(pred == y) * 100
return acc
if __name__ == "__main__":
params = parse_args("test")
pprint.pprint(params)
acc_all = []
iter_num = 600
few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot)
if params.dataset in ["omniglot", "cross_char"]:
assert (params.model == "Conv4" and not params.train_aug
), "omniglot only support Conv4 without augmentation"
params.model = "Conv4S"
if params.method == "baseline":
if ((epoch + 1) % params.save_freq == 0) or (epoch
== stop_epoch - 1):
outfile = os.path.join(params.checkpoint_dir,
'{:d}.tar'.format(epoch))
torch.save({
'epoch': epoch,
'state': model.state_dict()
}, outfile)
print("a epoch cost {} s".format(time.time() - start_epoch_time))
# return model
if __name__ == '__main__':
np.random.seed(10)
params = parse_args('train')
model = select_model(params)
base_loader, val_loader = select_dataloader_for_train(params)
model = model.cuda()
print_model_params(model, params)
params.checkpoint_dir = get_checkpoint_path(params)
if not os.path.isdir(params.checkpoint_dir):
os.makedirs(params.checkpoint_dir)
model, start_epoch, stop_epoch = load_presaved_model_for_train(
model, params)
json.dump(vars(params), open(params.checkpoint_dir + '/configs.json', 'w'))
def sum_up_results(self,
choose_by,
top_k,
show_same_params=True
): # choose the best according to dataset & split
def select_cols_if_exists(df, cols: list):
for col in cols.copy(): # BUGFIX: some of col not removed
if col not in list(df.columns):
logging.warning(
'sum_up_results()/"%s" not in dataframe, deleted from cols.'
% (col))
cols.remove(col)
return df[cols]
def del_all_the_same_cols(df):
if len(df.index) != 1:
# df = df[[col for col in df if not df[col].nunique()==1]]
df = df[[
col for col in df if not len(set(df[col])) == 1
]] # when only 2 unique and one is nan then this would work
else:
pass
return df
def replace_std_with_conf(df):
n_episodes = 600
df2 = df.copy()
df2['novel_acc_95%CI'] = 1.96 * df2['novel_acc_std'] / np.sqrt(
n_episodes)
del df2['novel_acc_std']
return df2
csv_path = os.path.join(self.record_folder,
self.fixed_params['test']['csv_name'])
print('sum_up_results/Reading file:', csv_path)
record_df = pd.read_csv(csv_path)
default_test_args = parse_args('test', parse_str='')
default_test_params = default_test_args.__dict__ # to avoid including other exps
important_fixed_params = {
**default_test_params,
**self.base_params,
**self.fixed_params['test']
}
# delete negligible_vars & changeable vars
del_keys(important_fixed_params, self.negligible_vars)
del_keys(important_fixed_params,
self.possible_params['general'].keys())
del_keys(important_fixed_params, self.possible_params['test'].keys())
# for multi-frac_ensemble
test_possible_params = copy.deepcopy(self.possible_params['test'])
if 'frac_ensemble' in test_possible_params.keys():
frac_param = test_possible_params['frac_ensemble']
if len(frac_param) == 1 and isinstance(frac_param[0], str):
if ',' in frac_param[0]:
frac_ls = frac_param[0].split(',')
frac_ls = list(map(frac_ensemble_str2var, frac_ls))
test_possible_params['frac_ensemble'] = frac_ls
all_possible_params = {
**self.possible_params['general'],
**test_possible_params
}
matched_df = get_matched_df(important_fixed_params,
record_df,
possible_params=all_possible_params)
##### debug #####
# print('matched_df:\n%s'%(matched_df))
if top_k == None:
top_k = len(matched_df)
if len(matched_df) != 0:
sorted_df = matched_df.sort_values(by=choose_by, ascending=False)
compare_cols = list(self.possible_params['general'].keys()) + list(
self.possible_params['test'].keys())
# compare_cols = compare_cols + ['epoch', 'train_acc_mean', 'val_acc_mean', 'novel_acc_mean']
compare_cols = compare_cols + [
'epoch', 'train_acc_mean', 'source_val_acc', 'val_acc_mean',
'novel_acc_mean', 'novel_acc_std'
]
print()
print('Best Test Acc: %s, selected by %s' %
(sorted_df['novel_acc_mean'].iloc[0], choose_by))
show_df = select_cols_if_exists(sorted_df, compare_cols)
show_df = del_all_the_same_cols(show_df)
if 'novel_acc_std' in show_df.columns:
# print('computing 95% confidence interval...')
show_df = replace_std_with_conf(show_df)
print()
print(
'=' * 20, 'Top %s/%s results sorted by: %s' %
(top_k, len(matched_df), choose_by), '=' * 20)
print(show_df.head(top_k))
# print(sorted_df[compare_cols].head(top_k))
else:
print('=' * 20, 'No experiment matching the conditions', '=' * 20)
acc_all.append((correct_this / count_this * 100))
###############################################################################################
pbar.set_postfix(avg_acc=np.mean(np.asarray(acc_all)))
acc_all = np.asarray(acc_all)
acc_mean = np.mean(acc_all)
acc_std = np.std(acc_all)
print('%d Test Acc = %4.2f%% +- %4.2f%%' %
(iter_num, acc_mean, 1.96 * acc_std / np.sqrt(iter_num)))
if __name__ == '__main__':
np.random.seed(10)
params = parse_args('test')
##################################################################
image_size = 224
iter_num = 600
n_query = max(
1, int(16 * params.test_n_way / params.train_n_way)
) #if test_n_way is smaller than train_n_way, reduce n_query to keep batch size small
few_shot_params = dict(n_way=params.test_n_way,
n_support=params.n_test_shot)
freeze_backbone = params.freeze_backbone
##################################################################
dataset_names = ["ISIC", "EuroSAT", "CropDisease", "ChestX"]
def main():
params = parse_args('test')
if params.method == "metatrain":
tester = MetaTrain(params)
func = getattr(tester, params.test, func_not_found)
func()
