def save_features(model, losses_bag, data_loader, features_dir, params): penultimate_dict, features_dict = evaluate(data_loader, model, losses_bag, params, save_latent=True) save_pickle(os.path.join(features_dir, 'novel_penultimate.plk'), penultimate_dict) save_pickle(os.path.join(features_dir, 'novel_features.plk'), features_dict) if __name__ == '__main__': params = parse_args('graph') random.seed(914637) image_size = 84 save_dir = './weights' data_dir = { 'cifar': './filelists/cifar/', 'CUB': './filelists/CUB/', 'miniImagenet': './filelists/miniImagenet/' } novel_file = data_dir[params.dataset] + 'novel.json' params.checkpoint_dir = '%s/checkpoints/%s/%s/%s' % ( save_dir, params.dataset, params.model, params.run_name) n_way, n_shot, n_val = params.n_way, params.n_shot, params.n_val novel_datamgr = SetDataManager(novel_file, image_size, n_way, n_shot,
if all_feats is None: all_feats = f.create_dataset('all_feats', [max_count] + list(feats.size()[1:]), dtype='f') all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy() all_labels[count:count + feats.size(0)] = y.cpu().numpy() count = count + feats.size(0) count_var = f.create_dataset('count', (1, ), dtype='i') count_var[0] = count f.close() if __name__ == '__main__': params = parse_args('save_features') assert params.method != 'maml' and params.method != 'maml_approx', 'maml do not support save_feature and run' if 'Conv' in params.model: if params.dataset in ['omniglot', 'cross_char']: image_size = 28 else: image_size = 84 else: image_size = 224 if params.dataset in ['omniglot', 'cross_char']: assert params.model == 'Conv4' and not params.train_aug, 'omniglot only support Conv4 without augmentation' params.model = 'Conv4S' split = params.split
model.eval() correct, total = 0, 0 for i, (x, y) in enumerate(testloader): x, y = x.cuda(), y.cuda() # print(y) x = pgd.attack(model, x, y) scores, _ = model.forward(x) predicted = torch.argmax(scores, 1) correct += (predicted == y).sum().item() total += predicted.size(0) print('Accuracy {:f}'.format(100. * correct / total)) if __name__ == '__main__': np.random.seed(10) params = parse_args() print(params) if params.dataset == 'cifar10': params.num_classes = 10 transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])
acc_all_ori = np.asarray(acc_all_ori) acc_mean_ori = np.mean(acc_all_ori) acc_std_ori = np.std(acc_all_ori) print('BSR+ENT: %d Test Acc = %4.2f%% +- %4.2f%%' % (iter_num, acc_mean_ori, 1.96 * acc_std_ori / np.sqrt(iter_num))) acc_all_lp = np.asarray(acc_all_lp) acc_mean_lp = np.mean(acc_all_lp) acc_std_lp = np.std(acc_all_lp) print('BSR+LP+ENT: %d Test Acc = %4.2f%% +- %4.2f%%' % (iter_num, acc_mean_lp, 1.96 * acc_std_lp / np.sqrt(iter_num))) if __name__ == '__main__': np.random.seed(10) params = parse_args('finetune') image_size = 224 iter_num = 600 params.method = 'bsr' few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot, n_query=15) freeze_backbone = params.freeze_backbone if params.dtarget == 'ISIC': print("Loading ISIC") datamgr = ISIC_few_shot.SetDataManager(image_size, n_eposide=iter_num, **few_shot_params)
def exp_grid(self, choose_by='val_acc_mean', mode='from_scratch'): ''' Args: mode (str): 'from_scratch'|'resume'|'draw_tasks'|'tmp_pkl' # 'draw_tasks' could be deprecated??? ''' print('exp_grid() start.') print('self.base_params:', self.base_params) base_params = self.base_params.copy() # set to test_n_way = default(5) if mode == 'tmp_pkl': if 'test_n_way' in base_params: print('change base_params["test_n_way"] to be 5.') base_params['test_n_way'] = 5 default_args = {} # the raw default args of the code default_args['train'] = parse_args('train', parse_str='') default_args['test'] = parse_args('test', parse_str='') possible_params = self.possible_params if mode == 'tmp_pkl': for key in possible_params['general']: if len(possible_params['general'][key]) != 1: raise ValueError( 'general param length should be 1 for mode: tmp_pkl') for key in possible_params['test']: if len(possible_params['test'][key]) != 1: raise ValueError( 'test param length should be 1 for mode: tmp_pkl') train_args = get_modified_args(default_args['train'], { **base_params, **self.fixed_params['train'] }) test_args = get_modified_args(default_args['test'], { **self.base_params, **self.fixed_params['test'] }) # test_n_way should be 2 all_general_possible_params = get_all_params_comb( possible_params=possible_params['general']) all_test_possible_params = get_all_params_comb( possible_params=possible_params['test']) csv_path = os.path.join(self.record_folder, self.fixed_params['test']['csv_name']) # TODO: refactor to many functions with argument: mode is_csv_exists = os.path.exists(csv_path) ########## deal with initial csv ########## if mode == 'from_scratch': if not is_csv_exists: # create empty csv print("Creating file: '%s' since not exist" % (csv_path)) with open(csv_path, "w") as file: empty_df = pd.DataFrame(columns=['dataset']) empty_df.to_csv(file, header=True, index=False) pkl_postfix_str = '_' + self.pkl_postfix #+ '.pkl' pkl_postfix_str += '.pkl' pkl_path = csv_path.replace( '.csv', pkl_postfix_str) # restore ALL experiments in this class is_pkl_exists = os.path.exists(pkl_path) ########## load pickle data ########## if mode == 'resume': print('loading self.results_pkl from:', pkl_path) if is_pkl_exists: with open(pkl_path, 'rb') as handle: self.results_pkl = pickle.load(handle) else: logging.warning( 'No previous result pickle file: %s, only record self.results_pkl from scratch.' % (pkl_path)) print('self.results_pkl at begin, len =', len(self.results_pkl)) elif mode == 'tmp_pkl': if is_pkl_exists: print('pickle file: %s, already exists.' % (pkl_path)) return for general_params in all_general_possible_params: ########## decide if should_train ########## if mode in ['resume', 'from_scratch']: # if mode == 'resume':# or mode == 'draw_tasks': print() print('=' * 20, 'Checking if already trained in:', csv_path, '=' * 20) print('base_params:', base_params) print('general_params:', general_params) # read csv loaded_df = pd.read_csv(csv_path) # check if ALL test_params has been experimented. if so, then do NOT even train model check_df = loaded_df.copy() check_param = {**base_params, **general_params} # include default columns if param not in dataframe default_train_args = parse_args('train', parse_str='') default_train_param = vars(default_train_args) for param in check_param: if param not in check_df.columns: print('param:', param) if default_train_param[param] is None: check_df[param] = np.nan else: check_df[param] = default_train_param[param] check_df = get_matched_df(check_param, check_df) num_experiments = len(check_df) should_train = num_experiments == 0 # elif mode == 'from_scratch': # should_train = True elif mode in ['draw_tasks', 'tmp_pkl']: should_train = False ########## training ########## if should_train: # release cuda memory torch.cuda.empty_cache() # train model print() print('=' * 20, 'Training', '=' * 20) print('general_params:', general_params) modified_train_args = get_modified_args( train_args, general_params) if mode == 'resume': modified_train_args.resume = True args_sanity_check(modified_train_args, script='train') train_result = exp_train_val(modified_train_args) else: if mode == 'draw_tasks': print('NO need to train when draw_tasks.') elif mode == 'resume': print('NO need to train since already trained in record:', csv_path) print('Summing up current record...') choose_by = 'val_acc_mean' top_k = None self.sum_up_results(choose_by=choose_by, top_k=top_k) else: print('NO need to train since already trained in record:', csv_path) modified_test_args = get_modified_args(test_args, general_params) # loop over testing settings under each general setting for test_params in all_test_possible_params: ########## check if should test ########## if mode == 'resume': print('\n', '=' * 20, 'Checking if already did experiments', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) check_df = loaded_df.copy() constrained_param = { **self.base_params, **general_params, **test_params } # add default params, delete useless params & dependent variables (result) default_args = parse_args('test', parse_str='') default_param = vars(default_args) check_param = {**default_param, **constrained_param} # delete vars not for checking del_vars = self.negligible_vars + self.dependent_vars for var in del_vars: if var in check_param: del check_param[var] # include default columns if param not in dataframe for param in check_param: if param not in check_df.columns: print('param:', param) if default_param[param] is None: check_df[param] = np.nan else: check_df[param] = default_param[param] # print(check_df) # yooooo check_df = get_matched_df( check_param, check_df) # should loaded_df be check_df ??? num_test_experiments = len(check_df) if num_test_experiments > 0: # already experiments print( 'NO need to test since already tested %s times in record: %s' % (num_test_experiments, csv_path)) print(check_df) continue else: print( 'Need to do testing since there\'s NO experiment in data.' ) elif mode == 'tmp_pkl': pass final_test_args = get_modified_args(modified_test_args, test_params) write_record = {**general_params, **test_params} ########## write train_acc to record dict ########## if mode in ['resume', 'from_scratch']: # if mode == 'resume': #in ['resume', 'draw_tasks']: source_val = True # write source_val_acc if source_val and 'cross' in base_params[ 'dataset'] and base_params[ 'dataset'] != 'cross': # in ['cross_base20cl', 'cross_char', 'cross_char_half', 'cross_char_quarter', 'cross_char2', 'cross_char2_base1lang']: if should_train: write_record['source_val_acc'] = train_result[ 'source_val_acc'] else: # to get source_val_acc, so no need test_params check_df = loaded_df.copy() check_df = get_matched_df( { **self.base_params, **general_params }, check_df) write_record['source_val_acc'] = check_df[ 'source_val_acc'].iloc[0] else: write_record['source_val_acc'] = None # write train_acc if should_train: write_record['train_acc_mean'] = train_result[ 'train_acc'] else: # to get train_acc, so no need test_params check_df = loaded_df.copy() check_df = get_matched_df( { **self.base_params, **general_params }, check_df) write_record['train_acc_mean'] = check_df[ 'train_acc_mean'].iloc[0] ########## save_features & test ########## if mode in ['from_scratch', 'resume', 'tmp_pkl']: ########### judge if should do several exp for frac_ensemble ########### tmp = final_test_args # to make variable name shorter is_str_single_frac = isinstance( tmp.frac_ensemble, str) and ',' not in tmp.frac_ensemble is_float_none_frac = isinstance( tmp.frac_ensemble, float ) or tmp.frac_ensemble == 1 or tmp.frac_ensemble is None is_single_exp = is_float_none_frac or is_str_single_frac if is_str_single_frac: final_test_args.frac_ensemble = frac_ensemble_str2var( params.frac_ensemble) if is_single_exp: # common frac_ensemble splits = ['val', 'novel'] # temporary no 'train' for split in splits: # val, novel ##### get args ##### split_final_test_args = copy_args(final_test_args) split_final_test_args.split = split print('\n', '=' * 20, 'Saving Features', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) print('data split:', split) args_sanity_check(split_final_test_args, script='save_features') args_sanity_check(split_final_test_args, script='test') ########## save features ########## exp_save_features(copy_args(split_final_test_args)) print('\n', '=' * 20, 'Testing', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) print('data split:', split) n_episodes = 10 if split_final_test_args.debug or mode == 'draw_tasks' else 600 ########## testing and record to dict ########## exp_record, task_datas = exp_test( copy_args(split_final_test_args), n_episodes=n_episodes, should_del_features=True ) #, show_data=show_data) write_record['epoch'] = exp_record['epoch'] write_record[split + '_acc_mean'] = exp_record['acc_mean'] write_record[split + '_acc_std'] = exp_record['acc_std'] torch.cuda.empty_cache() ########## record to csv ########## if mode in ['from_scratch', 'resume']: print('Saving record to:', csv_path) record_to_csv(final_test_args, write_record, csv_path=csv_path) print('=' * 20, 'Current Experiments', '=' * 20) choose_by = 'val_acc_mean' top_k = None self.sum_up_results(choose_by, top_k) ########## record to pickle ########## write_record[ 'novel_task_datas'] = task_datas # currently ignore val_task_datas self.results_pkl.append(write_record) print('Saving self.results_pkl into:', pkl_path) with open(pkl_path, 'wb') as handle: pickle.dump(self.results_pkl, handle, protocol=pickle.HIGHEST_PROTOCOL) else: ########## multiple frac_ensemble ########## # make frac_ensemble list frac_ls = final_test_args.frac_ensemble.split(',') frac_ls = list(map(frac_ensemble_str2var, frac_ls)) print('frac_ls:', frac_ls) ########## check if each frac_ensemble already done in exps ########## if mode == 'resume': for frac in frac_ls.copy(): print( '\n', '=' * 20, 'Checking if already did "frac_ensemble" experiments', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) check_df = loaded_df.copy() constrained_param = { **self.base_params, **general_params, **test_params } # only add this line different from before constrained_param['frac_ensemble'] = frac # add default params, delete useless params & dependent variables (result) default_args = parse_args('test', parse_str='') default_param = vars(default_args) check_param = { **default_param, **constrained_param } # delete vars not for checking del_vars = self.negligible_vars + self.dependent_vars for var in del_vars: if var in check_param: del check_param[var] check_df = get_matched_df( check_param, loaded_df) num_test_experiments = len(check_df) if num_test_experiments > 0: # already experiments print( 'NO need to test since already tested %s times in record: %s' % (num_test_experiments, csv_path)) print(check_df) frac_ls.remove(frac) else: print( 'Need to do testing for frac_ensemble:', frac, 'since there\'s NO experiment in data.' ) ########## do exps for multi-frac_ensemble ########## if len(frac_ls) > 0: final_test_args.frac_ensemble = frac_ls ##### initialize record for different frac ##### frac_write_records = [ copy.deepcopy(write_record) for _ in range(len(frac_ls)) ] ##### start exps ##### splits = ['val', 'novel'] for split in splits: # val, novel ##### get args ##### split_final_test_args = copy_args( final_test_args) split_final_test_args.split = split print('\n', '=' * 20, 'Saving Features', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) print('data split:', split) ##### sanity check ##### args_sanity_check(split_final_test_args, script='save_features') args_sanity_check(split_final_test_args, script='test') ########## save features ########## exp_save_features( copy_args(split_final_test_args)) print('\n', '=' * 20, 'Testing', '=' * 20) print('general_params:', general_params) print('test_params:', test_params) print('data split:', split) n_episodes = 10 if split_final_test_args.debug or mode == 'draw_tasks' else 600 ########## testing and record to dict ########## ##### return n_frac exps results ##### frac_exp_records, frac_task_datas = exp_test( copy_args(split_final_test_args), n_episodes=n_episodes, should_del_features=True) for frac_id, frac in enumerate(frac_ls): frac_write_record = frac_write_records[ frac_id] exp_record = frac_exp_records[frac_id] frac_write_record['frac_ensemble'] = frac frac_write_record['epoch'] = exp_record[ 'epoch'] frac_write_record[ split + '_acc_mean'] = exp_record['acc_mean'] frac_write_record[ split + '_acc_std'] = exp_record['acc_std'] torch.cuda.empty_cache() ########## record n_frac exps to csv ########## if mode in ['from_scratch', 'resume']: print('Saving record to:', csv_path) for frac_id, frac in enumerate(frac_ls): frac_write_record = frac_write_records[ frac_id] tmp_test_args = copy_args(final_test_args) tmp_test_args.frac_ensemble = frac record_to_csv(tmp_test_args, frac_write_record, csv_path=csv_path) print('=' * 20, 'Current Experiments', '=' * 20) choose_by = 'val_acc_mean' top_k = None self.sum_up_results(choose_by, top_k) ########## record n_frac exps to pickle ########## for frac_id, frac in enumerate(frac_ls): frac_write_record = frac_write_records[frac_id] frac_write_record[ 'novel_task_datas'] = frac_task_datas[ frac_id] # currently ignore val_task_datas self.results_pkl.append(frac_write_record) print('Saving self.results_pkl into:', pkl_path) with open(pkl_path, 'wb') as handle: pickle.dump(self.results_pkl, handle, protocol=pickle.HIGHEST_PROTOCOL) else: print( '=' * 20, 'all frac_ensemble exps had been done before.', '=' * 20) torch.cuda.empty_cache() ########## sum up results ########## # TODO: can also loop dataset if mode in ['from_scratch', 'resume', 'draw_tasks']: for choose_by in ['val_acc_mean', 'novel_acc_mean']: # read csv to compare results top_k = None self.sum_up_results(choose_by, top_k) if mode == 'draw_tasks': print('loading self.results_pkl from:', pkl_path) best_model_all_tasks = get_best_all_tasks(pkl_path=pkl_path) print('sorting tasks...') sorted_tasks = sorted( best_model_all_tasks, key=lambda i: float(i['acc'])) # in ascending order print('Draw Worst Tasks...') save_img_folder = os.path.join(self.record_folder, 'imgs') draw_tasks( sorted_tasks, n_tasks=3, save_img_folder=save_img_folder, exp_postfix=self.pkl_postfix ) # utilize self.results_pkl, save all_tasks of best res
# 暂时弃用 from visualize_datapre import visualize_datapre from io_utils import parse_args import os, random from my_utils import visualize_support_imgs if __name__ == "__main__": params = parse_args('mytest') loadfile = os.path.join('filelists', params.test_dataset, 'novel.json') sub_meta, class_names = visualize_datapre(params, loadfile) random.seed(0) # params test_n_way = params.test_n_way test_n_shot = params.test_n_shot test_n_query = params.test_n_query image_size = params.image_size classes_id = sub_meta.keys() selected_classes_id = random.sample(classes_id, test_n_way) # random selected images_path selected_imgs = {} for i in selected_classes_id: sub_imgs = sub_meta[i] sub_selected_imgs = random.sample(sub_imgs, test_n_shot + test_n_query) selected_imgs[i] = sub_selected_imgs visualize_support_imgs(selected_imgs, class_names, test_n_shot, image_size)
------- None """ exp = Experiment(args) input_data, data_info = exp.get_input() model, method_type, encoding_strategy = exp.get_model() if not args['grid_search']: param_grid = {} if method_type != data_info["prediction_type"]: raise RuntimeError("Experiment failed! " "Dataset type is {} " "but method type is {}".format( data_info["prediction_type"], method_type)) generate_results(model=model, input_data=input_data, tmpdir=exp.tmpdir, target_name=data_info['target_name'], _id=args['_id'], mode=method_type, filename=data_info['filename'], categories=data_info['categories'], ordinals=data_info['ordinals'], encoding_strategy=encoding_strategy, param_grid=param_grid) if __name__ == "__main__": args, param_grid = parse_args() main(args, param_grid)
if all_feats is None: all_feats = f.create_dataset("all_feats", [max_count] + list(feats.size()[1:]), dtype="f") all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy() all_labels[count:count + feats.size(0)] = y.cpu().numpy() count = count + feats.size(0) count_var = f.create_dataset("count", (1, ), dtype="i") count_var[0] = count f.close() if __name__ == "__main__": params = parse_args("save_features") assert (params.method != "maml" and params.method != "maml_approx" ), "maml do not support save_feature and run" if "Conv" in params.model: if params.dataset in ["omniglot", "cross_char"]: image_size = 28 else: image_size = 84 else: image_size = 224 if params.dataset in ["omniglot", "cross_char"]: assert (params.model == "Conv4" and not params.train_aug ), "omniglot only support Conv4 without augmentation" params.model = "Conv4S"
acc = model.test_loop(val_loader) if acc > max_acc: # for baseline and baseline++, we don't use validation here so we let acc = -1 print("--> Best model! save...") max_acc = acc outfile = os.path.join(params.checkpoint_dir, 'best_model.tar') torch.save({'epoch': epoch, 'state': model.state_dict()}, outfile) if (epoch % params.save_freq == 0) or (epoch == stop_epoch - 1): outfile = os.path.join(params.checkpoint_dir, '{:d}.tar'.format(epoch)) torch.save({'epoch': epoch, 'state': model.state_dict()}, outfile) return model if __name__ == '__main__': params = parse_args('train') _set_seed(parse_args('train').seed) if params.dataset == 'cross': base_file = configs.data_dir['miniImagenet'] + 'all.json' val_file = configs.data_dir['CUB'] + 'val.json' elif params.dataset == 'cross_char': base_file = configs.data_dir['omniglot'] + 'noLatin.json' val_file = configs.data_dir['emnist'] + 'val.json' else: base_file = configs.data_dir[params.dataset] + 'base.json' val_file = configs.data_dir[params.dataset] + 'val.json' if 'Conv' in params.model: if params.dataset in ['omniglot', 'cross_char']: image_size = 28 else:
if all_feats is None: all_feats = f.create_dataset('all_feats', [max_count] + list(feats.size()[1:]), dtype='f') all_feats[count:count + feats.size(0)] = feats.data.cpu().numpy() all_labels[count:count + feats.size(0)] = y.cpu().numpy() count = count + feats.size(0) count_var = f.create_dataset('count', (1, ), dtype='i') count_var[0] = count f.close() if __name__ == '__main__': params = parse_args('eval') assert params.method != 'maml' and params.method != 'maml_approx', 'maml do not support save_feature and run' if 'Conv' in params.model: if params.dataset in ['omniglot', 'cross_char']: image_size = 28 else: image_size = 84 else: image_size = 224 split = params.split loadfile = configs.data_dir[params.dataset] + split + '.json' if params.checkpoint_dir == '': checkpoint_dir = '%s/checkpoints/%s/%s_%s' % (
z_all = torch.from_numpy(np.array(z_all)) model.n_query = n_query if adaptation: scores = model.set_forward_adaptation(z_all, is_feature=True) else: scores = model.set_forward(z_all, is_feature=True) pred = scores.data.cpu().numpy().argmax(axis=1) y = np.repeat(range(n_way), n_query) acc = np.mean(pred == y) * 100 return acc if __name__ == '__main__': args = parse_args('test') if args.seed is not None: torch.manual_seed(args.seed) acc_all = [] iter_num = 600 few_shot_args = dict(n_way=args.test_n_way, n_support=args.n_shot) l3_model = None # Always load vocab for compatibility if args.dataset == 'CUB': vocab = lang_utils.load_vocab(configs.lang_dir)
json.dump(metrics, fout, sort_keys=True, indent=4, separators=(',', ': ')) # If didn't train, save model anyways if stop_epoch == 0: outfile = os.path.join(args.checkpoint_dir, 'best_model.tar') torch.save({'epoch': stop_epoch, 'state': model.state_dict()}, outfile) return model if __name__ == '__main__': args = parse_args('train') if args.seed is not None: torch.manual_seed(args.seed) # I don't seed the np rng since dataset loading uses multiprocessing with random choices. # https://github.com/numpy/numpy/issues/9650 # Unavoidable undeterminism here for now if args.dataset == 'cross': base_file = configs.data_dir['miniImagenet'] + 'all.json' val_file = configs.data_dir['CUB'] + 'val.json' elif args.dataset == 'cross_char': base_file = configs.data_dir['omniglot'] + 'noLatin.json' val_file = configs.data_dir['emnist'] + 'val.json' else: base_file = configs.data_dir[args.dataset] + 'base.json'
z_all = torch.from_numpy(np.array(z_all)) model.n_query = n_query if adaptation: scores = model.set_forward_adaptation(z_all, is_feature=True) else: scores = model.set_forward(z_all, is_feature=True) pred = scores.data.cpu().numpy().argmax(axis=1) y = np.repeat(range(n_way), n_query) acc = np.mean(pred == y) * 100 return acc if __name__ == "__main__": params = parse_args("test") pprint.pprint(params) acc_all = [] iter_num = 600 few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_shot) if params.dataset in ["omniglot", "cross_char"]: assert (params.model == "Conv4" and not params.train_aug ), "omniglot only support Conv4 without augmentation" params.model = "Conv4S" if params.method == "baseline":
if ((epoch + 1) % params.save_freq == 0) or (epoch == stop_epoch - 1): outfile = os.path.join(params.checkpoint_dir, '{:d}.tar'.format(epoch)) torch.save({ 'epoch': epoch, 'state': model.state_dict() }, outfile) print("a epoch cost {} s".format(time.time() - start_epoch_time)) # return model if __name__ == '__main__': np.random.seed(10) params = parse_args('train') model = select_model(params) base_loader, val_loader = select_dataloader_for_train(params) model = model.cuda() print_model_params(model, params) params.checkpoint_dir = get_checkpoint_path(params) if not os.path.isdir(params.checkpoint_dir): os.makedirs(params.checkpoint_dir) model, start_epoch, stop_epoch = load_presaved_model_for_train( model, params) json.dump(vars(params), open(params.checkpoint_dir + '/configs.json', 'w'))
def sum_up_results(self, choose_by, top_k, show_same_params=True ): # choose the best according to dataset & split def select_cols_if_exists(df, cols: list): for col in cols.copy(): # BUGFIX: some of col not removed if col not in list(df.columns): logging.warning( 'sum_up_results()/"%s" not in dataframe, deleted from cols.' % (col)) cols.remove(col) return df[cols] def del_all_the_same_cols(df): if len(df.index) != 1: # df = df[[col for col in df if not df[col].nunique()==1]] df = df[[ col for col in df if not len(set(df[col])) == 1 ]] # when only 2 unique and one is nan then this would work else: pass return df def replace_std_with_conf(df): n_episodes = 600 df2 = df.copy() df2['novel_acc_95%CI'] = 1.96 * df2['novel_acc_std'] / np.sqrt( n_episodes) del df2['novel_acc_std'] return df2 csv_path = os.path.join(self.record_folder, self.fixed_params['test']['csv_name']) print('sum_up_results/Reading file:', csv_path) record_df = pd.read_csv(csv_path) default_test_args = parse_args('test', parse_str='') default_test_params = default_test_args.__dict__ # to avoid including other exps important_fixed_params = { **default_test_params, **self.base_params, **self.fixed_params['test'] } # delete negligible_vars & changeable vars del_keys(important_fixed_params, self.negligible_vars) del_keys(important_fixed_params, self.possible_params['general'].keys()) del_keys(important_fixed_params, self.possible_params['test'].keys()) # for multi-frac_ensemble test_possible_params = copy.deepcopy(self.possible_params['test']) if 'frac_ensemble' in test_possible_params.keys(): frac_param = test_possible_params['frac_ensemble'] if len(frac_param) == 1 and isinstance(frac_param[0], str): if ',' in frac_param[0]: frac_ls = frac_param[0].split(',') frac_ls = list(map(frac_ensemble_str2var, frac_ls)) test_possible_params['frac_ensemble'] = frac_ls all_possible_params = { **self.possible_params['general'], **test_possible_params } matched_df = get_matched_df(important_fixed_params, record_df, possible_params=all_possible_params) ##### debug ##### # print('matched_df:\n%s'%(matched_df)) if top_k == None: top_k = len(matched_df) if len(matched_df) != 0: sorted_df = matched_df.sort_values(by=choose_by, ascending=False) compare_cols = list(self.possible_params['general'].keys()) + list( self.possible_params['test'].keys()) # compare_cols = compare_cols + ['epoch', 'train_acc_mean', 'val_acc_mean', 'novel_acc_mean'] compare_cols = compare_cols + [ 'epoch', 'train_acc_mean', 'source_val_acc', 'val_acc_mean', 'novel_acc_mean', 'novel_acc_std' ] print() print('Best Test Acc: %s, selected by %s' % (sorted_df['novel_acc_mean'].iloc[0], choose_by)) show_df = select_cols_if_exists(sorted_df, compare_cols) show_df = del_all_the_same_cols(show_df) if 'novel_acc_std' in show_df.columns: # print('computing 95% confidence interval...') show_df = replace_std_with_conf(show_df) print() print( '=' * 20, 'Top %s/%s results sorted by: %s' % (top_k, len(matched_df), choose_by), '=' * 20) print(show_df.head(top_k)) # print(sorted_df[compare_cols].head(top_k)) else: print('=' * 20, 'No experiment matching the conditions', '=' * 20)
acc_all.append((correct_this / count_this * 100)) ############################################################################################### pbar.set_postfix(avg_acc=np.mean(np.asarray(acc_all))) acc_all = np.asarray(acc_all) acc_mean = np.mean(acc_all) acc_std = np.std(acc_all) print('%d Test Acc = %4.2f%% +- %4.2f%%' % (iter_num, acc_mean, 1.96 * acc_std / np.sqrt(iter_num))) if __name__ == '__main__': np.random.seed(10) params = parse_args('test') ################################################################## image_size = 224 iter_num = 600 n_query = max( 1, int(16 * params.test_n_way / params.train_n_way) ) #if test_n_way is smaller than train_n_way, reduce n_query to keep batch size small few_shot_params = dict(n_way=params.test_n_way, n_support=params.n_test_shot) freeze_backbone = params.freeze_backbone ################################################################## dataset_names = ["ISIC", "EuroSAT", "CropDisease", "ChestX"]
def main(): params = parse_args('test') if params.method == "metatrain": tester = MetaTrain(params) func = getattr(tester, params.test, func_not_found) func()