def sib_initialize(self, params, provide_original_image): novel_file = '%s/features/%s/%s/novel.hdf5' % ( configs.sib_dir, params.dataset, params.model) image_file_path = "img_paths_%s_%s_%s.npy" % ( params.dataset, params.model, params.method) if provide_original_image: self.cl_data_file, self.path_data_file = feat_loader.init_loader( novel_file, provide_original_image, image_file_path) else: self.cl_data_file = feat_loader.init_loader( novel_file, provide_original_image, image_file_path) self.image_size = 80
def simpleshot_initialize(self, params, provide_original_image): novel_file = '%s/features/%s/%s/novel.hdf5' % ( configs.simple_shot_dir, params.dataset, params.model) # novel_file = "/model/1154027137/ifsl_features/features/" + params.method + "_" + params.dataset + "_novel.hdf5" image_file_path = "img_paths_%s_%s_%s.npy" % ( params.dataset, params.model, params.method) if provide_original_image: self.cl_data_file, self.path_data_file = feat_loader.init_loader( novel_file, provide_original_image, image_file_path) else: self.cl_data_file = feat_loader.init_loader( novel_file, provide_original_image, image_file_path) self.image_size = 84
def test(self, split='novel', epoch=0): self.outfile = self.outfile_template % split if split == 'novel': self.save_feature(self.novel_loader) else: self.save_feature(self.val_loader) cl_data_file = feat_loader.init_loader(self.outfile) acc_all = [] for i in tqdm(range(self.params.test_epoch)): if self.params.fast_adapt: acc = self.model.fast_adapt(cl_data_file) else: acc = self.model.test_loop(cl_data_file) acc_all.append(acc) acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('%d Test Acc = %4.2f%% +- %4.2f%%' % (self.params.test_epoch, acc_mean, 1.96 * acc_std / np.sqrt(self.params.test_epoch))) if self.params.mode != 'test': self.writer.add_scalar('acc/%s_acc' % split, acc_mean, epoch) return acc_mean
def baseline_s2m2_initialize(self, params, provide_original_image): if params.dataset in ['omniglot', 'cross_char']: assert params.model == 'Conv4' and not params.train_aug, 'omniglot only support Conv4 without augmentation' params.model = 'Conv4S' dataset = params.dataset if params.dataset == "cross": dataset = "miniImagenet" checkpoint_dir = '%s/checkpoints/%s/%s_%s' % ( configs.save_dir, dataset, params.model, params.method) if params.train_aug: checkpoint_dir += '_aug' if params.method not in ['baseline', 'baseline++', 'S2M2_R']: checkpoint_dir += '_%dway_%dshot' % (params.train_n_way, params.n_shot) self.checkpoint_dir = checkpoint_dir split = params.split if params.save_iter != -1: split_str = split + "_" + str(params.save_iter) else: split_str = split # defaut split = novel, but you can also test base or val classes novel_file = os.path.join( checkpoint_dir.replace("checkpoints", "features"), split_str + ".hdf5") if params.dataset == "cross": novel_file = novel_file.replace("miniImagenet", "cross") file_path = "img_paths_%s_%s_%s.npy" % (params.dataset, params.model, params.method) if provide_original_image: self.cl_data_file, self.path_data_file = feat_loader.init_loader( novel_file, provide_original_image, file_path) else: self.cl_data_file = feat_loader.init_loader( novel_file, provide_original_image, file_path) self.image_size = 224 if params.method == "S2M2_R": self.image_size = 80
def single_test(params, results_logger): acc_all = [] iter_num = 600 few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot) if params.dataset in ['omniglot', 'cross_char']: assert params.model == 'Conv4' and not params.train_aug, 'omniglot only support Conv4 without augmentation' params.model = 'Conv4S' if params.method == 'baseline': model = BaselineFinetune(model_dict[params.model], **few_shot_params) elif params.method == 'baseline++': model = BaselineFinetune(model_dict[params.model], loss_type='dist', **few_shot_params) elif params.method == 'protonet': model = ProtoNet(model_dict[params.model], **few_shot_params) elif params.method == 'DKT': model = DKT(model_dict[params.model], **few_shot_params) elif params.method == 'matchingnet': model = MatchingNet(model_dict[params.model], **few_shot_params) elif params.method in ['relationnet', 'relationnet_softmax']: if params.model == 'Conv4': feature_model = backbone.Conv4NP elif params.model == 'Conv6': feature_model = backbone.Conv6NP elif params.model == 'Conv4S': feature_model = backbone.Conv4SNP else: feature_model = lambda: model_dict[params.model](flatten=False) loss_type = 'mse' if params.method == 'relationnet' else 'softmax' model = RelationNet(feature_model, loss_type=loss_type, **few_shot_params) elif params.method in ['maml', 'maml_approx']: backbone.ConvBlock.maml = True backbone.SimpleBlock.maml = True backbone.BottleneckBlock.maml = True backbone.ResNet.maml = True model = MAML(model_dict[params.model], approx=(params.method == 'maml_approx'), **few_shot_params) if params.dataset in ['omniglot', 'cross_char' ]: # maml use different parameter in omniglot model.n_task = 32 model.task_update_num = 1 model.train_lr = 0.1 else: raise ValueError('Unknown method') model = model.cuda() checkpoint_dir = '%s/checkpoints/%s/%s_%s' % ( configs.save_dir, params.dataset, params.model, params.method) if params.train_aug: checkpoint_dir += '_aug' if not params.method in ['baseline', 'baseline++']: checkpoint_dir += '_%dway_%dshot' % (params.train_n_way, params.n_shot) # modelfile = get_resume_file(checkpoint_dir) if not params.method in ['baseline', 'baseline++']: if params.save_iter != -1: modelfile = get_assigned_file(checkpoint_dir, params.save_iter) else: modelfile = get_best_file(checkpoint_dir) if modelfile is not None: tmp = torch.load(modelfile) model.load_state_dict(tmp['state']) else: print("[WARNING] Cannot find 'best_file.tar' in: " + str(checkpoint_dir)) split = params.split if params.save_iter != -1: split_str = split + "_" + str(params.save_iter) else: split_str = split if params.method in ['maml', 'maml_approx', 'DKT']: # maml do not support testing with feature if 'Conv' in params.model: if params.dataset in ['omniglot', 'cross_char']: image_size = 28 else: image_size = 84 else: image_size = 224 datamgr = SetDataManager(image_size, n_eposide=iter_num, n_query=15, **few_shot_params) if params.dataset == 'cross': if split == 'base': loadfile = configs.data_dir['miniImagenet'] + 'all.json' else: loadfile = configs.data_dir['CUB'] + split + '.json' elif params.dataset == 'cross_char': if split == 'base': loadfile = configs.data_dir['omniglot'] + 'noLatin.json' else: loadfile = configs.data_dir['emnist'] + split + '.json' else: loadfile = configs.data_dir[params.dataset] + split + '.json' novel_loader = datamgr.get_data_loader(loadfile, aug=False) if params.adaptation: model.task_update_num = 100 # We perform adaptation on MAML simply by updating more times. model.eval() acc_mean, acc_std = model.test_loop(novel_loader, return_std=True) else: novel_file = os.path.join( checkpoint_dir.replace("checkpoints", "features"), split_str + ".hdf5" ) # defaut split = novel, but you can also test base or val classes cl_data_file = feat_loader.init_loader(novel_file) for i in range(iter_num): acc = feature_evaluation(cl_data_file, model, n_query=15, adaptation=params.adaptation, **few_shot_params) acc_all.append(acc) acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('%d Test Acc = %4.2f%% +- %4.2f%%' % (iter_num, acc_mean, 1.96 * acc_std / np.sqrt(iter_num))) with open('record/results.txt', 'a') as f: timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime()) aug_str = '-aug' if params.train_aug else '' aug_str += '-adapted' if params.adaptation else '' if params.method in ['baseline', 'baseline++']: exp_setting = '%s-%s-%s-%s%s %sshot %sway_test' % ( params.dataset, split_str, params.model, params.method, aug_str, params.n_shot, params.test_n_way) else: exp_setting = '%s-%s-%s-%s%s %sshot %sway_train %sway_test' % ( params.dataset, split_str, params.model, params.method, aug_str, params.n_shot, params.train_n_way, params.test_n_way) acc_str = '%d Test Acc = %4.2f%% +- %4.2f%%' % ( iter_num, acc_mean, 1.96 * acc_std / np.sqrt(iter_num)) f.write('Time: %s, Setting: %s, Acc: %s \n' % (timestamp, exp_setting, acc_str)) results_logger.log("single_test_acc", acc_mean) results_logger.log("single_test_acc_std", 1.96 * acc_std / np.sqrt(iter_num)) results_logger.log("time", timestamp) results_logger.log("exp_setting", exp_setting) results_logger.log("acc_str", acc_str) return acc_mean
loadfile = configs.data_dir['emnist'] + split + '.json' else: loadfile = configs.data_dir[params.dataset] + split + '.json' novel_loader = datamgr.get_data_loader(loadfile, aug=False) if params.adaptation: model.task_update_num = 100 #We perform adaptation on MAML simply by updating more times. model.eval() acc_mean, acc_std = model.test_loop(novel_loader, return_std=True) else: novel_file = os.path.join( checkpoint_dir.replace("checkpoints", "features"), split_str + ".hdf5" ) #defaut split = novel, but you can also test base or val classes cl_data_file = feat_loader.init_loader(novel_file) for i in range(iter_num): acc = feature_evaluation(cl_data_file, model, n_query=15, adaptation=params.adaptation, **few_shot_params) acc_all.append(acc) acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('%d Test Acc = %4.2f%% +- %4.2f%%' % (iter_num, acc_mean, 1.96 * acc_std / np.sqrt(iter_num))) with open('./record/results.txt', 'a') as f:
model.eval() acc_mean, acc_std = model.test_loop( novel_loader, return_std = True) else: rootdir = '/home/rajshekd/projects/FSG/PRODA/features' NovelF = ['omniglot/Conv4S_protonet_5way_5shot/vanila/noLatin.hdf5', 'omniglot/Conv4S_protonet_5way_5shot/vanila/novel.hdf5', 'cross_char/Conv4S_protonet_5way_5shot/vanila-Protonet/novel.hdf5'] all_z = [] for novel_file in NovelF: all_perm_ids_iter = [] select_class_iter = [] print(novel_file) # import ipdb # ipdb.set_trace() cl_data_file = feat_loader.init_loader(os.path.join(rootdir,novel_file)) for i in range(iter_num): z_set, select_class, all_perm_ids = feature_evaluation(cl_data_file, model, n_query = 15) all_perm_ids_iter.append(all_perm_ids) select_class_iter.append(select_class) all_z.append(z_set) # final file cl_data_file = feat_loader.init_loader( os.path.join( rootdir, 'cross_char/Conv4S_protonet_5way_5shot/adversarial-concatZ_domainReg-0.1_lr-0.0001_endEpoch-4000_DiscM-2FC512/novel.hdf5')) for i in range(iter_num): z_set, select_class, all_perm_ids = feature_evaluation(cl_data_file, model, n_query=15, all_perm_ids = all_perm_ids_iter[i], select_class = select_class_iter[i]) all_z.append(z_set) all_z = np.vstack(all_z)
split_str = split # model = get_model(params) # few_shot_params = dict(n_way = params.test_n_way , n_support = params.n_shot) # if params.gpu_id: # set_gpu_id(params.gpu_id) # device = torch.device('cuda:'+str(params.gpu_id)) # model = model.cuda() # else: # device = None # model = to_device(model) # draw whole data distribution or task sample distribution? feat_file = os.path.join( checkpoint_dir.replace("checkpoints","features"), split_str +".hdf5") cl_data_file = feat_loader.init_loader(feat_file) # dictionary, keys: class_num, content: list of features ''' features for Conv4, dim = 1600 = 64*5*5 ''' feat_list = [] label_list = [] num_classes = len(cl_data_file.keys()) # useless cls_transform = {} # transform class_id to 0~draw_n_classes-1 to draw color draw_n_classes = params.d_classes draw_samples_per_class = params.d_samples draw_class_indices = np.random.permutation(list(cl_data_file.keys()))[:draw_n_classes] for i, cls_idx in enumerate(draw_class_indices): cls_transform[cls_idx] = i cls_feats = cl_data_file[cls_idx] cls_n_samples = len(cls_feats)
try: model.load_state_dict(tmp['state']) except RuntimeError: print('warning! RuntimeError when load_state_dict()!') model.load_state_dict(tmp['state'], strict=False) except KeyError: for k in tmp['model_state']: ##### revise latter if 'running' in k: tmp['model_state'][k] = tmp['model_state'][k].squeeze() model.load_state_dict(tmp['model_state'], strict=False) except: raise # load feature file print(' load saved feature file') cl_data_file = feat_loader.init_loader(featurefile) # start evaluate print(' evaluate') for i in range(iter_num): acc = feature_evaluation(cl_data_file, model, n_query=15, **few_shot_params) acc_all.append(acc) # statics print(' get statics') acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all)
import torch import numpy as np import random import matplotlib.pyplot as plt import seaborn as sns import data.feature_loader as feat_loader from sklearn.manifold import TSNE sns.set_context("notebook", font_scale=1.1) sns.set_style("ticks") len_test = 20 len_type = 5 novel_file = "./novel.hdf5" novel_both_file = "./novel_both.hdf5" features = feat_loader.init_loader(novel_file) features_both = feat_loader.init_loader(novel_both_file) class_list = features.keys() select_class = random.sample(class_list, len_type) print(class_list) #select_class = [84,85,86,87,88,89, 90, 91] print(select_class) z_all = [] z_both_all = [] for cl in select_class: img_feat = features[cl] img_both_feat = features_both[cl] print('len of cases: ', len(img_feat))
model.load_state_dict(tmp['state']) # except RuntimeError: # print('warning! RuntimeError when load_state_dict()!') # model.load_state_dict(tmp['state'], strict=False) # except KeyError: # for k in tmp['model_state']: ##### revise latter # if 'running' in k: # tmp['model_state'][k] = tmp['model_state'][k].squeeze() # model.load_state_dict(tmp['model_state'], strict=False) except: raise # load feature file print(' load saved feature file') cl_data_file = feat_loader.init_loader(f) # start evaluate print(' evaluate') for i in range(iter_num): acc = feature_evaluation(cl_data_file, model, n_query=5, **few_shot_params) acc_all.append(acc) # statics print(' get statics') acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all)
model.task_update_num = 100 #We perform adaptation on MAML simply by updating more times. model.eval() acc_mean, acc_std = model.test_loop( novel_loader, return_std = True) else: #classes = [1, 20, 40, 60, 80]#1 # classes = [1, 10, 30, 50, 70]#2 # classes = [1, 20, 50, 80, 97]#3 classes = [1, 23, 53, 83, 93]#4 data = json.load(open('/home/rajshekd/projects/FSG/PRODA/filelists/vgg_flower/all.json')) imlist = [] # adv novel_file = "/home/rajshekd/projects/FSG/PRODA/features/CUB_flowers/ResNet10_protonet_aug_5way_5shot/adversarial-ConcatZ_domainReg-0.1_lr-0.0001_DiscM-4096_Base2Base/all.hdf5" cl_data_file1 = feat_loader.init_loader(novel_file) acc1_avg = [] for _ in range(600): acc1, supp, query, classes = feature_evaluation(cl_data_file1, model, select_class=classes) acc1_avg.append(acc1) acc1_avg = np.mean(acc1_avg) print ("accuracy: ",acc1_avg) # vanila novel_file = "/home/rajshekd/projects/FSG/PRODA/features/CUB_flowers/ResNet10_protonet_aug_5way_5shot/vanila/all.hdf5" cl_data_file2 = feat_loader.init_loader(novel_file) # ipdb.set_trace() acc2_avg = [] for _ in range(600): acc2, _, _, classes = feature_evaluation(cl_data_file2, model, select_class=classes) acc2_avg.append(acc2) acc2_avg = np.mean(acc2_avg)
def exp_test(params, n_episodes, should_del_features=False):#, show_data=False): start_time = datetime.datetime.now() print('exp_test() started at',start_time) set_random_seed(0) # successfully reproduce "normal" testing. if params.gpu_id: set_gpu_id(params.gpu_id) # acc_all = [] model = get_model(params, 'test') ########## get settings ########## n_shot = params.test_n_shot if params.test_n_shot is not None else params.n_shot few_shot_params = dict(n_way = params.test_n_way , n_support = n_shot) if params.gpu_id: model = model.cuda() else: model = to_device(model) checkpoint_dir = get_checkpoint_dir(params) print('loading from:',checkpoint_dir) if params.save_iter != -1: modelfile = get_assigned_file(checkpoint_dir, params.save_iter) else: modelfile = get_best_file(checkpoint_dir) ########## load model ########## if modelfile is not None: if params.gpu_id is None: tmp = torch.load(modelfile) else: # TODO: figure out WTF is going on here print('params.gpu_id =', params.gpu_id) map_location = 'cuda:0' # gpu_str = 'cuda:' + '0'#str(params.gpu_id) # map_location = {'cuda:1':gpu_str, 'cuda:0':gpu_str} # see here: https://hackmd.io/koKAo6kURn2YBqjoXXDhaw#RuntimeError-CUDA-error-invalid-device-ordinal tmp = torch.load(modelfile, map_location=map_location) # tmp = torch.load(modelfile) if not params.method in ['baseline', 'baseline++'] : # if 'baseline' or 'baseline++' then NO NEED to load model !!! model.load_state_dict(tmp['state']) print('Model successfully loaded.') else: print('No need to load model for baseline/baseline++ when testing.') load_epoch = int(tmp['epoch']) ########## testing ########## if params.method in ['maml', 'maml_approx']: #maml do not support testing with feature image_size = get_img_size(params) load_file = get_loadfile_path(params, params.split) datamgr = SetDataManager(image_size, n_episode = n_episodes, n_query = 15 , **few_shot_params) novel_loader = datamgr.get_data_loader( loadfile, aug = False) if params.adaptation: model.task_update_num = 100 #We perform adaptation on MAML simply by updating more times. model.eval() acc_mean, acc_std = model.test_loop( novel_loader, return_std = True) ########## last record and post-process ########## torch.cuda.empty_cache() timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime()) # TODO afterward: compute this acc_str = '%4.2f%% +- %4.2f%%' % (acc_mean, 1.96* acc_std/np.sqrt(n_episodes)) # writing settings into csv acc_mean_str = '%4.2f' % (acc_mean) acc_std_str = '%4.2f' %(acc_std) # record beyond params extra_record = {'time':timestamp, 'acc_mean':acc_mean_str, 'acc_std':acc_std_str, 'epoch':load_epoch} if should_del_features: del_features(params) end_time = datetime.datetime.now() print('exp_test() start at', start_time, ', end at', end_time, '.\n') print('exp_test() totally took:', end_time-start_time) return extra_record, task_datas else: # not MAML acc_all = [] # # draw_task: initialize task acc(actually can replace acc_all), img_path, img_is_correct, etc. # task_datas = [None]*n_episodes # list of dict # directly use extracted features all_feature_files = get_all_feature_files(params) if params.n_test_candidates is None: # common setting (no candidate) # draw_task: initialize task acc(actually can replace acc_all), img_path, img_is_correct, etc. task_datas = [None]*n_episodes # list of dict feature_file = all_feature_files[0] cl_feature, cl_filepath = feat_loader.init_loader(feature_file, return_path=True) cl_feature_single = [cl_feature] for i in tqdm(range(n_episodes)): # TODO afterward: fix data list? can only fix class list? task_data = feature_evaluation( cl_feature_single, model, params=params, n_query=15, **few_shot_params, cl_filepath=cl_filepath, ) acc = task_data['acc'] acc_all.append(acc) task_datas[i] = task_data acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('loaded from %d epoch model.' %(load_epoch)) print('%d episodes, Test Acc = %4.2f%% +- %4.2f%%' %(n_episodes, acc_mean, 1.96* acc_std/np.sqrt(n_episodes))) ########## last record and post-process ########## torch.cuda.empty_cache() timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime()) # TODO afterward: compute this acc_str = '%4.2f%% +- %4.2f%%' % (acc_mean, 1.96* acc_std/np.sqrt(n_episodes)) # writing settings into csv acc_mean_str = '%4.2f' % (acc_mean) acc_std_str = '%4.2f' %(acc_std) # record beyond params extra_record = {'time':timestamp, 'acc_mean':acc_mean_str, 'acc_std':acc_std_str, 'epoch':load_epoch} if should_del_features: del_features(params) end_time = datetime.datetime.now() print('exp_test() start at', start_time, ', end at', end_time, '.\n') print('exp_test() totally took:', end_time-start_time) return extra_record, task_datas else: # n_test_candidates settings candidate_cl_feature = [] # features of each class of each candidates print('Loading features of %s candidates into dictionaries...' %(params.n_test_candidates)) for n in tqdm(range(params.n_test_candidates)): nth_feature_file = all_feature_files[n] cl_feature, cl_filepath = feat_loader.init_loader(nth_feature_file, return_path=True) candidate_cl_feature.append(cl_feature) print('Evaluating...') # TODO: frac_acc_all is_single_exp = not isinstance(params.frac_ensemble, list) if is_single_exp: # draw_task: initialize task acc(actually can replace acc_all), img_path, img_is_correct, etc. task_datas = [None]*n_episodes # list of dict ########## test and record acc ########## for i in tqdm(range(n_episodes)): # TODO afterward: fix data list? can only fix class list? task_data = feature_evaluation( candidate_cl_feature, model, params=params, n_query=15, **few_shot_params, cl_filepath=cl_filepath, ) acc = task_data['acc'] acc_all.append(acc) task_datas[i] = task_data collected = gc.collect() # print("Garbage collector: collected %d objects." % (collected)) acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('loaded from %d epoch model.' %(load_epoch)) print('%d episodes, Test Acc = %4.2f%% +- %4.2f%%' %(n_episodes, acc_mean, 1.96* acc_std/np.sqrt(n_episodes))) collected = gc.collect() print("garbage collector: collected %d objects." % (collected)) ########## last record and post-process ########## torch.cuda.empty_cache() timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime()) # TODO afterward: compute this acc_str = '%4.2f%% +- %4.2f%%' % (acc_mean, 1.96* acc_std/np.sqrt(n_episodes)) # writing settings into csv acc_mean_str = '%4.2f' % (acc_mean) acc_std_str = '%4.2f' %(acc_std) # record beyond params extra_record = {'time':timestamp, 'acc_mean':acc_mean_str, 'acc_std':acc_std_str, 'epoch':load_epoch} if should_del_features: del_features(params) end_time = datetime.datetime.now() print('exp_test() start at', start_time, ', end at', end_time, '.\n') print('exp_test() totally took:', end_time-start_time) return extra_record, task_datas else: ########## multi-frac_ensemble exps ########## ########## (haven't modified) test and record acc ########## n_fracs = len(params.frac_ensemble) ##### initialize frac_data ##### frac_acc_alls = [[0]*n_episodes for _ in range(n_fracs)] frac_acc_means = [None]*n_fracs frac_acc_stds = [None]*n_fracs # draw_task: initialize task acc(actually can replace acc_all), img_path, img_is_correct, etc. ep_task_data_each_frac = [[None]*n_episodes for _ in range(n_fracs)] # list of list of dict for ep_id in tqdm(range(n_episodes)): # TODO afterward: fix data list? can only fix class list? # TODO my_utils.py: feature_eval return frac_task_data frac_task_data = feature_evaluation( candidate_cl_feature, model, params=params, n_query=15, **few_shot_params, cl_filepath=cl_filepath, ) for frac_id in range(n_fracs): task_data = frac_task_data[frac_id] # TODO: i think here's the problem??? acc = task_data['acc'] frac_acc_alls[frac_id][ep_id] = acc ep_task_data_each_frac[frac_id][ep_id] = task_data collected = gc.collect() # print("Garbage collector: collected %d objects." % (collected)) collected = gc.collect() # print("Garbage collector: collected %d objects." % (collected)) ### debug # print('frac_acc_alls:', frac_acc_alls) # yee for frac_id in range(n_fracs): frac_acc_alls[frac_id] = np.asarray(frac_acc_alls[frac_id]) acc_all = frac_acc_alls[frac_id] acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) frac_acc_means[frac_id] = acc_mean frac_acc_stds[frac_id] = acc_std print('loaded from %d epoch model, frac_ensemble:'%(load_epoch), params.frac_ensemble[frac_id]) print('%d episodes, Test Acc = %4.2f%% +- %4.2f%%' %(n_episodes, acc_mean, 1.96* acc_std/np.sqrt(n_episodes))) ########## (haven't modified) last record and post-process ########## torch.cuda.empty_cache() timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime()) # TODO afterward: compute this # acc_str = '%4.2f%% +- %4.2f%%' % (acc_mean, 1.96* acc_std/np.sqrt(n_episodes)) frac_extra_records = [] for frac_id in range(n_fracs): # writing settings into csv acc_mean = frac_acc_means[frac_id] acc_std = frac_acc_stds[frac_id] acc_mean_str = '%4.2f' % (acc_mean) acc_std_str = '%4.2f' %(acc_std) # record beyond params extra_record = {'time':timestamp, 'acc_mean':acc_mean_str, 'acc_std':acc_std_str, 'epoch':load_epoch} frac_extra_records.append(extra_record) if should_del_features: del_features(params) end_time = datetime.datetime.now() print('exp_test() start at', start_time, ', end at', end_time, '.\n') print('exp_test() totally took:', end_time-start_time) return frac_extra_records, ep_task_data_each_frac
params = parse_args('test') few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot) checkpoint_dir = '%s/checkpoints/%s/%s_%s' % ( configs.save_dir, params.dataset, params.model, params.method) if params.train_aug: checkpoint_dir += '_aug' if not params.method in ['baseline', 'baseline++']: checkpoint_dir += '_%dway_%dshot' % (params.train_n_way, params.n_shot) split = params.split if params.save_iter != -1: split_str = split + "_" + str(params.save_iter) else: split_str = split novel_file = os.path.join( checkpoint_dir.replace("checkpoints", "features"), split_str + ".hdf5") outs = [ './features/miniImagenet/ResNet34_baseline_aug/novel.hdf5', './features/miniImagenet/ResNet34_baseline_aug/base.hdf5', './features/miniImagenet/ResNet34_baseline_aug/val.hdf5' ] base_file = outs[1] val_file = outs[2] base_data_file = feat_loader.init_loader(base_file) val_data_file = feat_loader.init_loader(val_file) agent_train(base_data_file, val_data_file, n_query=15, **few_shot_params)
def get_logits_targets(params): acc_all = [] iter_num = 600 few_shot_params = dict(n_way = params.test_n_way , n_support = params.n_shot) if params.dataset in ['omniglot', 'cross_char']: assert params.model == 'Conv4' and not params.train_aug ,'omniglot only support Conv4 without augmentation' params.model = 'Conv4S' if params.method == 'baseline': model = BaselineFinetune( model_dict[params.model], **few_shot_params ) elif params.method == 'baseline++': model = BaselineFinetune( model_dict[params.model], loss_type = 'dist', **few_shot_params ) elif params.method == 'protonet': model = ProtoNet( model_dict[params.model], **few_shot_params ) elif params.method == 'DKT': model = DKT(model_dict[params.model], **few_shot_params) elif params.method == 'matchingnet': model = MatchingNet( model_dict[params.model], **few_shot_params ) elif params.method in ['relationnet', 'relationnet_softmax']: if params.model == 'Conv4': feature_model = backbone.Conv4NP elif params.model == 'Conv6': feature_model = backbone.Conv6NP elif params.model == 'Conv4S': feature_model = backbone.Conv4SNP else: feature_model = lambda: model_dict[params.model]( flatten = False ) loss_type = 'mse' if params.method == 'relationnet' else 'softmax' model = RelationNet( feature_model, loss_type = loss_type , **few_shot_params ) elif params.method in ['maml' , 'maml_approx']: backbone.ConvBlock.maml = True backbone.SimpleBlock.maml = True backbone.BottleneckBlock.maml = True backbone.ResNet.maml = True model = MAML( model_dict[params.model], approx = (params.method == 'maml_approx') , **few_shot_params ) if params.dataset in ['omniglot', 'cross_char']: #maml use different parameter in omniglot model.n_task = 32 model.task_update_num = 1 model.train_lr = 0.1 else: raise ValueError('Unknown method') model = model.cuda() checkpoint_dir = '%s/checkpoints/%s/%s_%s' %(configs.save_dir, params.dataset, params.model, params.method) if params.train_aug: checkpoint_dir += '_aug' if not params.method in ['baseline', 'baseline++'] : checkpoint_dir += '_%dway_%dshot' %( params.train_n_way, params.n_shot) #modelfile = get_resume_file(checkpoint_dir) if not params.method in ['baseline', 'baseline++'] : if params.save_iter != -1: modelfile = get_assigned_file(checkpoint_dir,params.save_iter) else: modelfile = get_best_file(checkpoint_dir) if modelfile is not None: tmp = torch.load(modelfile) model.load_state_dict(tmp['state']) else: print("[WARNING] Cannot find 'best_file.tar' in: " + str(checkpoint_dir)) split = params.split if params.save_iter != -1: split_str = split + "_" +str(params.save_iter) else: split_str = split if params.method in ['maml', 'maml_approx', 'DKT']: #maml do not support testing with feature if 'Conv' in params.model: if params.dataset in ['omniglot', 'cross_char']: image_size = 28 else: image_size = 84 else: image_size = 224 datamgr = SetDataManager(image_size, n_eposide = iter_num, n_query = 15 , **few_shot_params) if params.dataset == 'cross': if split == 'base': loadfile = configs.data_dir['miniImagenet'] + 'all.json' else: loadfile = configs.data_dir['CUB'] + split +'.json' elif params.dataset == 'cross_char': if split == 'base': loadfile = configs.data_dir['omniglot'] + 'noLatin.json' else: loadfile = configs.data_dir['emnist'] + split +'.json' else: loadfile = configs.data_dir[params.dataset] + split + '.json' novel_loader = datamgr.get_data_loader( loadfile, aug = False) if params.adaptation: model.task_update_num = 100 #We perform adaptation on MAML simply by updating more times. model.eval() logits_list = list() targets_list = list() for i, (x,_) in enumerate(novel_loader): logits = model.get_logits(x).detach() targets = torch.tensor(np.repeat(range(params.test_n_way), model.n_query)).cuda() logits_list.append(logits) #.cpu().detach().numpy()) targets_list.append(targets) #.cpu().detach().numpy()) else: novel_file = os.path.join( checkpoint_dir.replace("checkpoints","features"), split_str +".hdf5") cl_data_file = feat_loader.init_loader(novel_file) logits_list = list() targets_list = list() n_query = 15 n_way = few_shot_params['n_way'] n_support = few_shot_params['n_support'] class_list = cl_data_file.keys() for i in range(iter_num): #---------------------- select_class = random.sample(class_list,n_way) z_all = [] for cl in select_class: img_feat = cl_data_file[cl] perm_ids = np.random.permutation(len(img_feat)).tolist() z_all.append( [ np.squeeze( img_feat[perm_ids[i]]) for i in range(n_support+n_query) ] ) # stack each batch z_all = torch.from_numpy(np.array(z_all)) model.n_query = n_query logits = model.set_forward(z_all, is_feature = True).detach() targets = torch.tensor(np.repeat(range(n_way), n_query)).cuda() logits_list.append(logits) targets_list.append(targets) #---------------------- return torch.cat(logits_list, 0), torch.cat(targets_list, 0)
# init sub_json_name, sub_json_path = produce_subjson_file( selected_classes_id, sub_meta, meta, params) # set the shuffle False to recognize the support sample sub_datamgr = SimpleDataManager(image_size, batch_size=params.test_bs, isAircraft=isAircraft, shuffle=False) sub_data_loader = sub_datamgr.get_data_loader(sub_json_path, aug=False) output_path = checkpoint_dir_test.replace("checkpoints", "features") output_name = sub_json_name.replace(".json", ".hdf5") output_file = Path(os.path.join(output_path, output_name)).as_posix() # pass if hdf5 file already exists if not os.path.exists(output_file): save_features(model, sub_data_loader, output_file) sub_novel_file = output_file # sub_novel_file = os.path.join( checkpoint_dir_test.replace("checkpoints","features"), split_str +"_shuffle_false.hdf5") print('load novel file from:', sub_novel_file) sub_cl_data_file = feat_loader.init_loader(sub_novel_file) acc, class_acc = feature_evaluation(sub_cl_data_file, model, n_query=65, adaptation=params.adaptation, **few_shot_params) print_class_acc(class_acc, class_names)
if __name__ == '__main__': params = parse_args('test') few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot) checkpoint_dir = '%s/checkpoints/%s/%s_%s' % ( configs.save_dir, params.dataset, params.model, params.method) if params.train_aug: checkpoint_dir += '_aug' if not params.method in ['baseline', 'baseline++']: checkpoint_dir += '_%dway_%dshot' % (params.train_n_way, params.n_shot) split = params.split if params.save_iter != -1: split_str = split + "_" + str(params.save_iter) else: split_str = split novel_file = os.path.join( checkpoint_dir.replace("checkpoints", "features"), split_str + ".hdf5") outs = [ './features/miniImagenet/ResNet34_baseline_aug/novel.hdf5', './features/miniImagenet/ResNet34_baseline_aug/base.hdf5', './features/miniImagenet/ResNet34_baseline_aug/val.hdf5' ] test_file = outs[0] test_data_file = feat_loader.init_loader(test_file) agent_test(test_data_file, n_query=15, **few_shot_params)