def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) if n_gpus>1: maml = torch.nn.DataParallel(maml) state_dict = torch.load(Param.out_path+args.ckpt) print(state_dict.keys()) pretrained_dict = OrderedDict() for k in state_dict.keys(): if n_gpus==1: pretrained_dict[k[7:]] = deepcopy(state_dict[k]) else: pretrained_dict[k[0:]] = deepcopy(state_dict[k]) maml.load_state_dict(pretrained_dict) print("Load from ckpt:", Param.out_path+args.ckpt) #opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) #trainset = MiniImagenet(Param.root, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) #valset = MiniImagenet(Param.root, mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) #trainloader = DataLoader(trainset, batch_size=args.task_num, shuffle=True, num_workers=4, drop_last=True) #valloader = DataLoader(valset, batch_size=4, shuffle=True, num_workers=4, drop_last=True) testloader = DataLoader(testset, batch_size=4, shuffle=True, num_workers=4, drop_last=True) #train_data = inf_get(trainloader) #val_data = inf_get(valloader) test_data = inf_get(testloader) """Test for 600 epochs (each has 4 tasks)""" ans = None maml_clone = deepcopy(maml) for itr in range(600): # 600x4 test tasks support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to(Param.device), support_y.to(Param.device), qx.to(Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train = False) if(ans is None): ans = temp else: ans = torch.cat([ans, temp], dim = 0) if itr%100==0: print(itr,ans.mean(dim = 0).tolist()) ans = ans.mean(dim = 0).tolist() print('Acc: '+str(ans)) with open(Param.out_path+'test.json','w') as f: json.dump(ans,f)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5])] device = torch.device('cuda') maml = Meta(args, config).to(device) ckpt_dir = "./checkpoint_miniimage.pth" print("Load trained model") ckpt = torch.load(ckpt_dir) maml.load_state_dict(ckpt['model']) mini_test = MiniImagenet("F:\\ACV_project\\MAML-Pytorch\\miniimagenet\\", mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=1, resize=args.imgsz) db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] #count = 0 #print("Test_loader",db_test) for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): torch.manual_seed(222) # 为cpu设置种子,为了使结果是确定的 torch.cuda.manual_seed_all(222) # 为GPU设置种子,为了使结果是确定的 np.random.seed(222) print(args) config = [ ('conv2d', [32, 1, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 7040]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) if os.path.exists( "./models/" + str("./models/miniimagenet_maml" + str(args.n_way) + "way_" + str(args.k_spt) + "shot.pkl")): path = "./models/" + str("./models/miniimagenet_maml" + str(args.n_way) + "way_" + str(args.k_spt) + "shot.pkl") maml.load_state_dict(path) print("load model success") tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet("./miniimagenet", mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000) mini_test = MiniImagenet("./miniimagenet", mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100) test_accuracy = [] for epoch in range(10): # fetch meta_batchsz num of episode each time db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs, loss_t = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs) test_accuracy.append(accs[-1]) average_accuracy = sum(test_accuracy) / len(test_accuracy) print("average accuracy:{}".format(average_accuracy))
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) #np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) if len(args.gpu.split(',')) > 1: maml = torch.nn.DataParallel(maml) state_dict = torch.load(Param.out_path + args.ckpt) print(state_dict.keys()) pretrained_dict = OrderedDict() for k in state_dict.keys(): if n_gpus == -1: pretrained_dict[k[7:]] = deepcopy(state_dict[k]) else: pretrained_dict[k[0:]] = deepcopy(state_dict[k]) maml.load_state_dict(pretrained_dict) print("Load from ckpt:", Param.out_path + args.ckpt) #opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) if args.loader in [0, 1]: # default loader if args.loader == 1: #from dataloader.mini_imagenet import MiniImageNet as MiniImagenet from MiniImagenet2 import MiniImagenet else: from MiniImagenet import MiniImagenet testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testloader = DataLoader(testset, batch_size=1, shuffle=False, num_workers=4, worker_init_fn=worker_init_fn, drop_last=True) test_data = inf_get(testloader) elif args.loader == 2: # pkl loader args_data = {} args_data['x_dim'] = "84,84,3" args_data['ratio'] = 1.0 args_data['seed'] = 222 loader_test = dataset_mini(600, 100, 'test', args_data) loader_test.load_data_pkl() """Test for 600 epochs (each has 4 tasks)""" ans = None maml_clone = deepcopy(maml) for itr in range(600): # 600x4 test tasks if args.loader in [0, 1]: support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) elif args.loader == 2: support_x, support_y, qx, qy = get_data(loader_test) support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train=False) if (ans is None): ans = temp else: ans = torch.cat([ans, temp], dim=0) if itr % 100 == 0: print(itr, ans.mean(dim=0).tolist()) meanacc = np.array(ans.mean(dim=0).tolist()) stdacc = np.array(ans.std(dim=0).tolist()) ci95 = 1.96 * stdacc / np.sqrt(600) print(f'Acc: {meanacc[-1]:.4f}, ci95: {ci95[-1]:.4f}') with open(Param.out_path + 'test.txt', 'w') as f: print(f'Acc: {meanacc[-1]:.4f}, ci95: {ci95[-1]:.4f}', file=f)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) ## Task Learner Setup task_config = [('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5])] last_epoch = 0 suffix = "_v0" save_path = os.getcwd() + '/data/model_batchsz' + str( args.k_model) + '_stepsz' + str( args.update_lr) + '_epoch' + str(last_epoch) + suffix + '.pt' while os.path.isfile(save_path): valid_epoch = last_epoch last_epoch += 500 save_path = os.getcwd() + '/data/model_batchsz' + str( args.k_model) + '_stepsz' + str( args.update_lr) + '_epoch' + str(last_epoch) + suffix + '.pt' save_path = os.getcwd() + '/data/model_batchsz' + str( args.k_model) + '_stepsz' + str( args.update_lr) + '_epoch' + str(valid_epoch) + suffix + '.pt' device = torch.device('cuda') task_mod = Meta(args, task_config).to(device) task_mod.load_state_dict(torch.load(save_path)) task_mod.eval() ## AL Learner Setup print(args) al_config = [('linear', [1, 32 * 5 * 5])] device = torch.device('cuda') maml = AL_Learner(args, al_config, task_mod).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet('/home/tesca/data/miniimagenet/', mode='train', n_way=args.n_way, k_shot=1, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) mini_test = MiniImagenet('/home/tesca/data/miniimagenet/', mode='test', n_way=args.n_way, k_shot=1, k_query=args.k_qry, batchsz=100, resize=args.imgsz) save_path = os.getcwd() + '/data/model_batchsz' + str( args.k_model) + '_stepsz' + str(args.update_lr) + '_epoch' for epoch in range(args.epoch // 10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) al_accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('step:', step, '\tAL acc:', al_accs) if step % 500 == 0: # evaluation torch.save(maml.state_dict(), save_path + str(step) + "_al_net.pt") '''db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True)
def main(args): step = args.step set_seed(args.seed) adj, features, labels = load_citation(args.dataset, args.normalization) features = sgc_precompute(features, adj, args.degree) if args.dataset == 'citeseer': node_num = 3327 class_label = [0, 1, 2, 3, 4, 5] combination = list(combinations(class_label, 2)) elif args.dataset == 'cora': node_num = 2708 class_label = [0, 1, 2, 3, 4, 5, 6] combination = list(combinations(class_label, 2)) config = [('linear', [args.hidden, features.size(1)]), ('linear', [args.n_way, args.hidden])] device = torch.device('cuda') for i in range(len(combination)): print("Cross Validation: {}".format((i + 1))) maml = Meta(args, config).to(device) test_label = list(combination[i]) train_label = [n for n in class_label if n not in test_label] print('Cross Validation {} Train_Label_List: {} '.format( i + 1, train_label)) print('Cross Validation {} Test_Label_List: {} '.format( i + 1, test_label)) for j in range(args.epoch): x_spt, y_spt, x_qry, y_qry = sgc_data_generator( features, labels, node_num, train_label, args.task_num, args.n_way, args.k_spt, args.k_qry) accs = maml.forward(x_spt, y_spt, x_qry, y_qry) print('Step:', j, '\tMeta_Training_Accuracy:', accs) if j % 100 == 0: torch.save(maml.state_dict(), 'maml.pkl') meta_test_acc = [] for k in range(step): model_meta_trained = Meta(args, config).to(device) model_meta_trained.load_state_dict(torch.load('maml.pkl')) model_meta_trained.eval() x_spt, y_spt, x_qry, y_qry = sgc_data_generator( features, labels, node_num, test_label, args.task_num, args.n_way, args.k_spt, args.k_qry) accs = model_meta_trained.forward(x_spt, y_spt, x_qry, y_qry) meta_test_acc.append(accs) if args.dataset == 'citeseer': with open('citeseer.txt', 'a') as f: f.write( 'Cross Validation:{}, Step: {}, Meta-Test_Accuracy: {}' .format( i + 1, j, np.array(meta_test_acc).mean(axis=0).astype( np.float16))) f.write('\n') elif args.dataset == 'cora': with open('cora.txt', 'a') as f: f.write( 'Cross Validation:{}, Step: {}, Meta-Test_Accuracy: {}' .format( i + 1, j, np.array(meta_test_acc).mean(axis=0).astype( np.float16))) f.write('\n')
def main(args): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [] if args.arch == "Unet": for block in range(args.NUM_DOWN_CONV): out_channels = (2**block) * args.HIDDEN_DIM if (block == 0): config += [( 'conv2d', [out_channels, args.imgc, 3, 3, 1, 1] ) # out_c, in_c, k_h, k_w, stride, padding, also only conv, without bias ] else: config += [ ('conv2d', [out_channels, out_channels // 2, 3, 3, 1, 1]), # out_c, in_c, k_h, k_w, stride, padding ] config += [ ('leakyrelu', [0.2, False]), # alpha; if true then executes relu in place ('bn', [out_channels]) ] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('max_pool2d', [2, 2, 0])] # kernel_size, stride, padding for block in range(args.NUM_DOWN_CONV - 1): out_channels = (2**(args.NUM_DOWN_CONV - block - 2)) * args.HIDDEN_DIM in_channels = out_channels * 3 config += [('upsample', [2])] config += [('conv2d', [out_channels, in_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [ ('conv2d_b', [args.outc, args.HIDDEN_DIM, 3, 3, 1, 1]) ] # all the conv2d before are without bias, and this conv_b is with bias else: raise ("architectures other than Unet hasn't been added!!") device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) # print(maml) for name, param in maml.named_parameters(): print(name, param.size()) print('Total trainable tensors:', num) SUMMARY_INTERVAL = 5 TEST_PRINT_INTERVAL = SUMMARY_INTERVAL * 5 ITER_SAVE_INTERVAL = 300 EPOCH_SAVE_INTERVAL = 5 model_path = "/scratch/users/chenkaim/pytorch-models/pytorch_" + args.model_name + "_k_shot_" + str( args.k_spt) + "_task_num_" + str(args.task_num) + "_meta_lr_" + str( args.meta_lr) + "_inner_lr_" + str( args.update_lr) + "_num_inner_updates_" + str(args.update_step) if not os.path.isdir(model_path): os.mkdir(model_path) start_epoch = 0 if (args.continue_train): print("Restoring weights from ", model_path + "/epoch_" + str(args.continue_epoch) + ".pt") checkpoint = torch.load(model_path + "/epoch_" + str(args.continue_epoch) + ".pt") print(checkpoint.keys()) print(checkpoint.items()) maml.load_state_dict(checkpoint['state_dict']) maml.lr_scheduler.load_state_dict(checkpoint['scheduler']) maml.meta_optim.load_state_dict(checkpoint['optimizer']) start_epoch = args.continue_epoch db = RCWA_data_loader(batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz, data_folder=args.data_folder) for step in range(start_epoch, args.epoch): print("epoch: ", step) if step % EPOCH_SAVE_INTERVAL == 0: torch.save(maml.state_dict(), model_path + "/epoch_" + str(step) + ".pt") for itr in range( int(0.7 * db.total_data_samples / ((args.k_spt + args.k_qry) * args.task_num))): x_spt, y_spt, x_qry, y_qry = db.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs, loss_q, ave_trans, min_trans = maml.transference( x_spt, y_spt, x_qry, y_qry) if itr % SUMMARY_INTERVAL == 0: print_str = "Iteration %d: pre-inner-loop train accuracy: %.5f, post-iner-loop test accuracy: %.5f, train_loss: %.5f, ave_trans: %.2f, min_trans: %.2f" % ( itr, accs[0], accs[-1], loss_q, ave_trans, min_trans) print(print_str) if itr % TEST_PRINT_INTERVAL == 0: accs = [] for _ in range(10): # test x_spt, y_spt, x_qry, y_qry = db.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip( x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print( 'Meta-validation pre-inner-loop train accuracy: %.5f, meta-validation post-inner-loop test accuracy: %.5f' % (accs[0], accs[-1])) maml.lr_scheduler.step()