def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) maml = torch.nn.DataParallel(maml) opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) trainset = MiniImagenet(Param.root, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) trainloader = DataLoader(trainset, batch_size=args.task_num, shuffle=True, num_workers=4, drop_last=True) testloader = DataLoader(testset, batch_size=4, shuffle=True, num_workers=4, drop_last=True) train_data = inf_get(trainloader) test_data = inf_get(testloader) for epoch in range(args.epoch): support_x, support_y, meta_x, meta_y = train_data.__next__() support_x, support_y, meta_x, meta_y = support_x.to(Param.device), support_y.to(Param.device), meta_x.to(Param.device), meta_y.to(Param.device) meta_loss = maml(support_x, support_y, meta_x, meta_y).mean() opt.zero_grad() meta_loss.backward() torch.nn.utils.clip_grad_value_(maml.parameters(), clip_value = 10.0) opt.step() plot.plot('meta_loss', meta_loss.item()) if(epoch % 2000 == 999): ans = None maml_clone = deepcopy(maml) for _ in range(600): support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to(Param.device), support_y.to(Param.device), qx.to(Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train = False) if(ans is None): ans = temp else: ans = torch.cat([ans, temp], dim = 0) ans = ans.mean(dim = 0).tolist() test_result[epoch] = ans if (ans[-1] > best_acc): best_acc = ans[-1] torch.save(maml.state_dict(), Param.out_path + 'net_'+ str(epoch) + '_' + str(best_acc) + '.pkl') del maml_clone print(str(epoch) + ': '+str(ans)) with open(Param.out_path+'test.json','w') as f: json.dump(test_result,f) if (epoch < 5) or (epoch % 100 == 99): plot.flush() plot.tick()
def main(): torch.manual_seed(1) torch.cuda.manual_seed_all(1) np.random.seed(1) print(args) config = [('conv2d', [args.num_filters, 1, 3, 3, 2, 1]), ('relu', [True]), ('bn', [args.num_filters]), ('conv2d', [args.num_filters, args.num_filters, 3, 3, 2, 1]), ('relu', [True]), ('bn', [args.num_filters]), ('conv2d', [args.num_filters, args.num_filters, 3, 3, 2, 1]), ('relu', [True]), ('bn', [args.num_filters]), ('conv2d', [args.num_filters, args.num_filters, 3, 3, 2, 1]), ('relu', [True]), ('bn', [args.num_filters]), ('flatten', []), ('linear', [args.n_way + 2, args.num_filters * 9])] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total sampled meta-task number if args.train == 'True': mini_train = LingualData('./data', mode='train', task_type=args.task_type, n_way=args.n_way, k_shot=args.k_spt_train, k_query=args.k_qry_train, k_unk_shot=args.k_spt_unk_train, k_unk_query=args.k_qry_unk_train, k_silence_shot=args.k_spt_silence_train, k_silence_query=args.k_qry_silence_train, batchsz=16000, resize=args.imgsz, unk_sil_spt=args.unk_sil_spt) exp_string = 'cls_' + str(args.n_way) + '.tskn_' + str( args.task_num) + '.spttrain_' + str( args.k_spt_train) + '.qrytrain_' + str( args.k_qry_train) + '.numstep' + str( args.update_step) + '.updatelr' + str(args.update_lr) model_path = args.logdir + '/' + exp_string model_file = None if args.train == 'True': if not os.path.exists(model_path): os.makedirs(model_path) print("logs directory ", args.logdir, " created!") writer = SummaryWriter(model_path) set_logger(os.path.join(args.logdir, 'train.log')) train(maml, mini_train, model_path, args.resume_itr, device, writer) else: if args.test_iter >= 0: model_file = model_path + '/' + 'model-' + str( args.test_iter) + '.pth' test(maml, model_file, device)
def main(args): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [('conv2d', [64, 1, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 2, 2, 1, 0]), ('relu', [True]), ('bn', [64]), ('flatten', []), ('linear', [args.n_way, 256])] #device = torch.device('cuda') device = torch.device('cpu') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) db_train = SvhnNShot(batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz) for step in range(args.epoch): x_spt, y_spt, x_qry, y_qry = db_train.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs = maml(x_spt, y_spt.long(), x_qry, y_qry.long()) if step % 50 == 0: print('step:', step, '\ttraining acc:', accs) if step % 500 == 0: accs = [] for _ in range(1000 // args.task_num): # test x_spt, y_spt, x_qry, y_qry = db_train.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip( x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one.long(), x_qry_one, y_qry_one.long()) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) if n_gpus>1: maml = torch.nn.DataParallel(maml) state_dict = torch.load(Param.out_path+args.ckpt) print(state_dict.keys()) pretrained_dict = OrderedDict() for k in state_dict.keys(): if n_gpus==1: pretrained_dict[k[7:]] = deepcopy(state_dict[k]) else: pretrained_dict[k[0:]] = deepcopy(state_dict[k]) maml.load_state_dict(pretrained_dict) print("Load from ckpt:", Param.out_path+args.ckpt) #opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) #trainset = MiniImagenet(Param.root, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) #valset = MiniImagenet(Param.root, mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) #trainloader = DataLoader(trainset, batch_size=args.task_num, shuffle=True, num_workers=4, drop_last=True) #valloader = DataLoader(valset, batch_size=4, shuffle=True, num_workers=4, drop_last=True) testloader = DataLoader(testset, batch_size=4, shuffle=True, num_workers=4, drop_last=True) #train_data = inf_get(trainloader) #val_data = inf_get(valloader) test_data = inf_get(testloader) """Test for 600 epochs (each has 4 tasks)""" ans = None maml_clone = deepcopy(maml) for itr in range(600): # 600x4 test tasks support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to(Param.device), support_y.to(Param.device), qx.to(Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train = False) if(ans is None): ans = temp else: ans = torch.cat([ans, temp], dim = 0) if itr%100==0: print(itr,ans.mean(dim = 0).tolist()) ans = ans.mean(dim = 0).tolist() print('Acc: '+str(ans)) with open(Param.out_path+'test.json','w') as f: json.dump(ans,f)
def main(): print(args) device = torch.device('cuda') maml = Meta(args).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) trainset = Gen(args.task_num, args.k_spt, args.k_qry) testset = Gen(args.task_num, args.k_spt, args.k_qry * 10) for epoch in range(args.epoch): ind = [i for i in range(trainset.xs.shape[0])] np.random.shuffle(ind) xs, ys = torch.Tensor(trainset.xs[ind]).to(device), torch.Tensor( trainset.ys[ind]).to(device) xq, yq = torch.Tensor(trainset.xq[ind]).to(device), torch.Tensor( trainset.yq[ind]).to(device) maml.train() loss = maml(xs, ys, xq, yq, epoch) print('Epoch: {} Initial loss: {} Train loss: {}'.format( epoch, loss[0] / args.task_num, loss[-1] / args.task_num)) if (epoch + 1) % 50 == 0: print("Evaling the model...") torch.save(maml.state_dict(), 'save.pt') # del(maml) # maml = Meta(args).to(device) # maml.load_state_dict(torch.load('save.pt')) maml.eval() i = random.randint(0, testset.xs.shape[0] - 1) xs, ys = torch.Tensor(testset.xs[i]).to(device), torch.Tensor( testset.ys[i]).to(device) xq, yq = torch.Tensor(testset.xq[i]).to(device), torch.Tensor( testset.yq[i]).to(device) losses, losses_q, logits_q, _ = maml.finetunning(xs, ys, xq, yq) print('Epoch: {} Initial loss: {} Test loss: {}'.format( epoch, losses_q[0], losses_q[-1]))
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [ ('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5]) ] cuda = 'cuda:' + args.gpu_index device = torch.device(cuda) maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) # print(maml) print('Total trainable tensors:', num) if args.mode == 0: mode_val_test = 'val' train = 'train' else: mode_val_test = 'test' train = 'train_ts' # batchsz here means total episode number mini = MiniImagenet('../flower/', mode=train, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz, cross_val_idx=args.cross_val_idx) mini_test = MiniImagenet('../flower/', mode=mode_val_test, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.imgsz, cross_val_idx=args.cross_val_idx) accs_list_tr = [] accs_list_ts = [] for epoch in range(args.epoch//10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('step:', step, '\ttraining acc:', accs) accs_list_tr.append(accs) if step % 500 == 0 or (step == 10000//args.task_num - 1) & (epoch == range(args.epoch//10000)[-1]): # evaluation db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('step:', step, '\ttest acc:', accs) accs_list_ts.append(accs) if (step == 10000//args.task_num - 1) & (epoch == range(args.epoch//10000)[-1]): with open('acc_cv2/natural2_' + mode_val_test + '(task_num_' + str(args.task_num) + ').txt', mode='a') as f: f.write(str(accs[-1]) + ',')
def main(): start_time = time.time() torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) np.random.seed(args.seed) print(args) print(argv) os.makedirs(args.modelfile.split('/')[0], exist_ok=True) config = [ ('conv2d', [32, 3, 3, 3, 1, 1]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) mini = MiniImagenet('./dataset/mini-imagenet/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) if args.domain == 'mini': mini_test = MiniImagenet('./dataset/mini-imagenet/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.test_iter, resize=args.imgsz) mini_val = MiniImagenet('./dataset/mini-imagenet/', mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.test_iter, resize=args.imgsz) elif args.domain == 'cub': print("CUB dataset") mini_test = MiniImagenet('./dataset/CUB_200_2011/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.test_iter, resize=args.imgsz) elif args.domain == 'traffic': print("Traffic dataset") mini_test = MiniImagenet('./dataset/GTSRB/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.test_iter, resize=args.imgsz) elif args.domain == 'flower': print("flower dataset") mini_test = MiniImagenet('./dataset/102flowers/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.test_iter, resize=args.imgsz) else: print("Dataset Error") return if args.mode == 'test': count = 0 db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=6, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: print(count) count += 1 x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry, 'test', args.modelfile, pertub_scale=args.pertub_scale, num_ensemble=args.num_ensemble, fgsm_epsilon=args.fgsm_epsilon) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) np.set_printoptions(linewidth=1000) print("Running Time:", time.time()-start_time) print(accs) return for epoch in range(args.epoch//10000): db = DataLoader(mini, args.task_num, shuffle=True, num_workers=4, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('epoch:', epoch, 'step:', step, '\ttraining acc:', accs) if step % 200 == 0: print("Save model", args.modelfile) torch.save(maml, args.modelfile) db_test = DataLoader(mini_val, 1, shuffle=True, num_workers=4, pin_memory=True) accs_all_val = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry, 'train_test') accs_all_val.append(accs) db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=4, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) accs_val = np.array(accs_all_val).mean(axis=0).astype(np.float16) save_modelfile = "{}_{}_{}_{:0.4f}_{:0.4f}.pth".format(args.modelfile, epoch, step, accs_val[-1], accs[-1]) print(save_modelfile) torch.save(maml, save_modelfile) print("Val:", accs_val) print("Test:", accs)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) # 5 layers num_filter = 128 config = [ ('conv2d', [num_filter, 103, 3, 3, 1, 1]), ('relu', [True]), ('bn', [num_filter]), ('max_pool2d', [2, 2, 0]), ('conv2d', [num_filter, num_filter, 3, 3, 1, 1]), ('relu', [True]), ('bn', [num_filter]), ('max_pool2d', [2, 2, 0]), ('conv2d', [num_filter, num_filter, 3, 3, 1, 1]), ('relu', [True]), ('bn', [num_filter]), ('max_pool2d', [2, 2, 0]), ('conv2d', [num_filter, num_filter, 3, 3, 1, 1]), ('relu', [True]), ('bn', [num_filter]), ('max_pool2d', [2, 2, 0]), ('conv2d', [num_filter, num_filter, 3, 3, 1, 1]), ('relu', [True]), ('bn', [num_filter]), ('flatten', []), ('linear', [args.n_way, num_filter]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) import h5py ########################## 训练数据集 ########################## f=h5py.File('./data/PU-16-16-103-train0-4-1000perclass.h5','r') data=f['data'][:] f.close() data=data.reshape(-1,16,16,103) data=data.transpose((0,3,1,2)) dataset_train=HSI_dataset(data, 10000, args.n_way, args.k_spt, args.k_qry, 5, 1000) model_loss = [] for epoch in range(1): db = DataLoader(dataset_train, args.task_num, shuffle=True, num_workers=0, pin_memory=True) S = time.time() for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(device), x_qry.to(device), y_qry.to(device) #print(y_qry.shape) # torch.Size([4, 60]) accs, loss = maml(x_spt, y_spt, x_qry, y_qry) if (step+1) % 5 == 0: print('epoch: ', epoch + 1, '\tstep: ', step + 1, '\ttraining acc:', accs, 'model loss:', loss) model_loss.append(loss) f = open('./results/PU_train_time_update10.txt', 'w') f.write('train_time: ' + str(time.time() - S)) f.close() '''
n_way = 5 k_shot = 1 k_query = 1 batchsz = 3 meta = Meta(n_way, k_shot).cuda() mdl_file = 'ckpt/meta%d%d.mdl' % (n_way, k_shot) pretrain_mdl_file = 'ckpt/pretrain.mdl' # if os.path.exists(mdl_file): # print('>>load checkpoint ...', mdl_file) # meta.load_state_dict(torch.load(mdl_file)) # elif os.path.exists(pretrain_mdl_file): # print('>>load pretrain ...', pretrain_mdl_file) # meta.load_state_dict(torch.load(pretrain_mdl_file)) model_parameters = filter(lambda p: p.requires_grad, meta.parameters()) params = sum([np.prod(p.size()) for p in model_parameters]) print('total params:', params) optimizer = optim.Adam(meta.parameters(), lr=1e-5, weight_decay=1e-4) scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', verbose=True) tb = SummaryWriter('runs', str(datetime.now())) best_val_acc = 0 for epoch in range(1000): mini = MiniImagenet('../mini-imagenet/', mode='train', n_way=n_way, k_shot=k_shot, k_query=k_query, batchsz=1000,
def main(): saver = Saver(args) # set log log_format = '%(asctime)s %(message)s' logging.basicConfig(level=logging.INFO, format=log_format, datefmt='%m/%d %I:%M:%S %p', filename=os.path.join(saver.experiment_dir, 'log.txt'), filemode='w') console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger().addHandler(console) if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True saver.create_exp_dir(scripts_to_save=glob.glob('*.py') + glob.glob('*.sh') + glob.glob('*.yml')) saver.save_experiment_config() summary = TensorboardSummary(saver.experiment_dir) writer = summary.create_summary() best_pred = 0 logging.info(args) device = torch.device('cuda') criterion = nn.CrossEntropyLoss() criterion = criterion.to(device) maml = Meta(args, criterion).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) logging.info(maml) logging.info('Total trainable tensors: {}'.format(num)) # batch_size here means total episode number mini = MiniImagenet(args.data_path, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batch_size=args.batch_size, resize=args.img_size) mini_valid = MiniImagenet(args.data_path, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batch_size=args.test_batch_size, resize=args.img_size) train_loader = DataLoader(mini, args.meta_batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) valid_loader = DataLoader(mini_valid, args.meta_test_batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) for epoch in range(args.epoch): # fetch batch_size num of episode each time logging.info('--------- Epoch: {} ----------'.format(epoch)) train_accs_theta, train_accs_w = meta_train(train_loader, maml, device, epoch, writer) logging.info( '[Epoch: {}]\t Train acc_theta: {}\t Train acc_w: {}'.format( epoch, train_accs_theta, train_accs_w)) test_accs_theta, test_accs_w = meta_test(valid_loader, maml, device, epoch, writer) logging.info( '[Epoch: {}]\t Test acc_theta: {}\t Test acc_w: {}'.format( epoch, test_accs_theta, test_accs_w)) genotype = maml.model.genotype() logging.info('genotype = %s', genotype) logging.info(F.softmax(maml.model.alphas_normal, dim=-1)) logging.info(F.softmax(maml.model.alphas_reduce, dim=-1)) # Save the best meta model. new_pred = test_accs_w[-1] if new_pred > best_pred: is_best = True best_pred = new_pred else: is_best = False saver.save_checkpoint( { 'epoch': epoch, 'state_dict_w': maml.module.state_dict() if isinstance(maml, nn.DataParallel) else maml.state_dict(), 'state_dict_theta': maml.model.arch_parameters(), 'best_pred': best_pred, }, is_best)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [ ('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet('F:\\ACV_project\\MAML-Pytorch\\miniimagenet\\', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) mini_test = MiniImagenet('F:\\ACV_project\\MAML-Pytorch\\miniimagenet\\', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.imgsz) ckpt_dir = "./model/" for epoch in range(args.epoch//10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('step:', step, '\ttraining acc:', accs) if step % 500 == 0: # evaluation db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs) # save checkpoints os.makedirs(ckpt_dir, exist_ok=True) print('Saving the model as a checkpoint...') torch.save({'epoch': epoch, 'Steps': step, 'model': maml.state_dict()}, os.path.join(ckpt_dir, 'checkpoint.pth'))
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) #np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) if len(args.gpu.split(',')) > 1: maml = torch.nn.DataParallel(maml) opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) if args.loader in [0, 1]: # default loader if args.loader == 1: #from dataloader.mini_imagenet import MiniImageNet as MiniImagenet from MiniImagenet2 import MiniImagenet else: from MiniImagenet import MiniImagenet trainset = MiniImagenet(Param.root, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) trainloader = DataLoader(trainset, batch_size=args.task_num, shuffle=True, num_workers=4, worker_init_fn=worker_init_fn, drop_last=True) testloader = DataLoader(testset, batch_size=1, shuffle=True, num_workers=1, worker_init_fn=worker_init_fn, drop_last=True) train_data = inf_get(trainloader) test_data = inf_get(testloader) elif args.loader == 2: # pkl loader args_data = {} args_data['x_dim'] = "84,84,3" args_data['ratio'] = 1.0 args_data['seed'] = 222 loader_train = dataset_mini(600, 100, 'train', args_data) #loader_val = dataset_mini(600, 100, 'val', args_data) loader_test = dataset_mini(600, 100, 'test', args_data) loader_train.load_data_pkl() #loader_val.load_data_pkl() loader_test.load_data_pkl() for epoch in range(args.epoch): np.random.seed() if args.loader in [0, 1]: support_x, support_y, meta_x, meta_y = train_data.__next__() support_x, support_y, meta_x, meta_y = support_x.to( Param.device), support_y.to(Param.device), meta_x.to( Param.device), meta_y.to(Param.device) elif args.loader == 2: support_x, support_y, meta_x, meta_y = get_data(loader_train) support_x, support_y, meta_x, meta_y = support_x.to( Param.device), support_y.to(Param.device), meta_x.to( Param.device), meta_y.to(Param.device) meta_loss = maml(support_x, support_y, meta_x, meta_y).mean() opt.zero_grad() meta_loss.backward() torch.nn.utils.clip_grad_value_(maml.parameters(), clip_value=10.0) opt.step() plot.plot('meta_loss', meta_loss.item()) if (epoch % 2500 == 0): ans = None maml_clone = deepcopy(maml) for _ in range(600): if args.loader in [0, 1]: support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) elif args.loader == 2: support_x, support_y, qx, qy = get_data(loader_test) support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train=False) if (ans is None): ans = temp else: ans = torch.cat([ans, temp], dim=0) ans = ans.mean(dim=0).tolist() test_result[epoch] = ans if (ans[-1] > best_acc): best_acc = ans[-1] torch.save( maml.state_dict(), Param.out_path + 'net_' + str(epoch) + '_' + str(best_acc) + '.pkl') del maml_clone print(str(epoch) + ': ' + str(ans)) with open(Param.out_path + 'test.json', 'w') as f: json.dump(test_result, f) if (epoch < 5) or (epoch % 100 == 0): plot.flush() plot.tick()
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5])] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet( '/mnt/aitrics_ext/ext01/yanbin/MAML-Pytorch/data/miniImagenet/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) mini_val = MiniImagenet( '/mnt/aitrics_ext/ext01/yanbin/MAML-Pytorch/data/miniImagenet/', mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=600, resize=args.imgsz) mini_test = MiniImagenet( '/mnt/aitrics_ext/ext01/yanbin/MAML-Pytorch/data/miniImagenet/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=600, resize=args.imgsz) best_acc = 0.0 if not os.path.exists('ckpt/{}'.format(args.exp)): os.mkdir('ckpt/{}'.format(args.exp)) for epoch in range(args.epoch // 10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 500 == 0: print('step:', step, '\ttraining acc:', accs) if step % 1000 == 0: # evaluation db_val = DataLoader(mini_val, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_val = [] for x_spt, y_spt, x_qry, y_qry in db_val: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_val.append(accs) mean, std, ci95 = cal_conf(np.array(accs_all_val)) print('Val acc:{}, std:{}. ci95:{}'.format( mean[-1], std[-1], ci95[-1])) if mean[-1] > best_acc or step % 5000 == 0: best_acc = mean[-1] torch.save( maml.state_dict(), 'ckpt/{}/model_e{}s{}_{:.4f}.pkl'.format( args.exp, epoch, step, best_acc)) with open('ckpt/' + args.exp + '/val.txt', 'a') as f: print( 'val epoch {}, step {}: acc_val:{:.4f}, ci95:{:.4f}' .format(epoch, step, best_acc, ci95[-1]), file=f) ## Test db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) mean, std, ci95 = cal_conf(np.array(accs_all_test)) print('Test acc:{}, std:{}, ci95:{}'.format( mean[-1], std[-1], ci95[-1])) with open('ckpt/' + args.exp + '/test.txt', 'a') as f: print( 'test epoch {}, step {}: acc_test:{:.4f}, ci95:{:.4f}' .format(epoch, step, mean[-1], ci95[-1]), file=f)
if __name__ == '__main__': opt = get_parser() if opt.cuda and not torch.cuda.is_available(): raise Exception("GPU is not found, please run without --cuda") cudnn.benchmark = True if opt.cuda: device = torch.device('cuda') else: device = torch.device('cpu') maml = Meta(opt).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # # batchsz here means total episode number mini = MiniImagenet('./miniimagenet/', mode='train', n_way=opt.n_way, k_shot=opt.k_spt, k_query=opt.k_qry, batchsz=10000, resize=opt.img_size) mini_test = MiniImagenet('./miniimagenet/', mode='test',
def main(): ck = util.checkpoint(args) seed = args.seed random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) np.random.seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) ck.write_log(str(args)) # t = str(int(time.time())) # t = args.save_name # os.mkdir('./{}'.format(t)) # (ch_out, ch_in, k, k, stride, padding) config = [('conv2d', [32, 16, 3, 3, 1, 1]), ('relu', [True]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('relu', [True]), ('+1', [True]), ('conv2d', [3, 32, 3, 3, 1, 1])] device = torch.device('cuda') maml = Meta(args, config).to(device) # (Dataset) calculate the number of trainable tensors tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) ck.write_log(str(maml)) ck.write_log('Total trainable tensors: {}'.format(num)) # (Dataset) batchsz here means total episode number DL_MSI = dl.StereoMSIDatasetLoader(args) db = DL_MSI.train_loader dv = DL_MSI.valid_loader psnr = [] l1_loss = [] psnr_valid = [] for epoch, (spt_ms, spt_rgb, qry_ms, qry_rgb) in enumerate(db): if epoch // args.epoch: break spt_ms, spt_rgb, qry_ms, qry_rgb = (spt_ms.to(device), spt_rgb.to(device), qry_ms.to(device), qry_rgb.to(device)) # optimization is carried out inside meta_learner class, maml. accs, train_loss = maml(spt_ms, spt_rgb, qry_ms, qry_rgb, epoch) maml.scheduler.step() if epoch % args.print_every == 0: log_epoch = 'epoch: {} \ttraining acc: {}'.format(epoch, accs) ck.write_log(log_epoch) psnr.append(accs) l1_loss.append(train_loss) ck.plot_loss(psnr, l1_loss, epoch, args.print_every) if epoch % args.save_every == 0: with torch.no_grad(): ck.save(maml.net, maml.meta_optim, epoch) eval_psnr = 0 # psnr loss for idx, (valid_ms, valid_rgb) in enumerate(dv): #print('idx', idx) valid_ms, valid_rgb = prepare([valid_ms, valid_rgb]) sr_rgb = maml.net(valid_ms) sr_rgb = torch.clamp(sr_rgb, 0, 1) eval_psnr += errors.find_psnr(valid_rgb, sr_rgb) ############## plot PSNR here you idiot! ########### psnr_valid.append(eval_psnr / 25) ck.plot_psnr(psnr_valid, epoch, args.save_every) ck.write_log('Max PSNR is: {}'.format(max(psnr_valid))) imsave( './{}/validation/img_{}.png'.format(ck.dir, epoch), np.uint8(sr_rgb[0, :, :, :].permute( 1, 2, 0).cpu().detach().numpy() * 255)) ck.done()
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) maml = torch.nn.DataParallel(maml) opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # Load pkl dataset args_data = {} args_data['x_dim'] = "84,84,3" args_data['ratio'] = 1.0 args_data['seed'] = 222 loader_train = dataset_mini(600, 100, 'train', args_data) #loader_val = dataset_mini(600, 100, 'val', args_data) loader_test = dataset_mini(600, 100, 'test', args_data) loader_train.load_data_pkl() #loader_val.load_data_pkl() loader_test.load_data_pkl() for epoch in range(args.epoch): support_x, support_y, meta_x, meta_y = get_data(loader_train) support_x, support_y, meta_x, meta_y = support_x.to( Param.device), support_y.to(Param.device), meta_x.to( Param.device), meta_y.to(Param.device) meta_loss = maml(support_x, support_y, meta_x, meta_y).mean() opt.zero_grad() meta_loss.backward() torch.nn.utils.clip_grad_value_(maml.parameters(), clip_value=10.0) opt.step() plot.plot('meta_loss', meta_loss.item()) if (epoch % 2000 == 999): ans = None maml_clone = deepcopy(maml) for _ in range(600): support_x, support_y, qx, qy = get_data(loader_test) support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train=False) if (ans is None): ans = temp else: ans = torch.cat([ans, temp], dim=0) ans = ans.mean(dim=0).tolist() test_result[epoch] = ans if (ans[-1] > best_acc): best_acc = ans[-1] torch.save( maml.state_dict(), Param.out_path + 'net_' + str(epoch) + '_' + str(best_acc) + '.pkl') del maml_clone print(str(epoch) + ': ' + str(ans)) with open(Param.out_path + 'test.json', 'w') as f: json.dump(test_result, f) if (epoch < 5) or (epoch % 100 == 99): plot.flush() plot.tick()
def main(): torch.manual_seed(666) torch.cuda.manual_seed_all(666) np.random.seed(666) # data_name = 'BlogCatalog' print(args) nb_epochs = 50 nb_runs = 16 nb_try = 16 nb_batch_maml = 10 nb_batch = 32 lr_1 = 0.03 lr_s = lr_1 * args.task_num tr = 0.6 # features, labels, idx_train, y_train, idx_val, y_val, idx_labeled, train_unlabeled = SGC_process(data_name, degree=2, l_ratio=0.08, tr_ratio=0.8) # print(args.task_num) aucfile = 'results/auc_' + datetime.now().strftime("%m_%d_%H_%M") + '_yelp.txt' with open(aucfile, 'a') as f: f.write("settings: {labeled ratio: %f, training ratio: %f, epochs: %d, update_step: %d}\n" % (lr_1, tr, nb_epochs, args.update_step)) for t in range(nb_try): taskData = task(nb_task=args.task_num, degree=2, l_ratio=lr_1, t_ratio=tr, name='yelp') taskData.loadNProcess() f.write("target data name:" + taskData.f_name[-1] + "\n") f.write("%d-th try: \n" % t) for i in range(nb_runs): # training maml print("maml training...") print("In %d-th run..." % (i + 1)) f.write("%d-th run\n" % i) feature_list, label, l_list, ul_list, idx_test = taskData.sampleAnomaly() config = modelArch(feature_list[0].shape[1], args.n_way) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") maml = Meta(args, config).to(device) # stats of parameters to be updated tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) # print(maml) print("Total #trainable tensors: ", num) batch_gen = DataLoaderN(feature_list, l_list, ul_list, b_size=8, b_size_qry=6, nb_task=args.task_num, device=device) maml.train() for e in range(1, nb_epochs + 1): print("Running %d-th epoch" % e) epoch_loss = 0 epoch_acc = 0 for b in range(nb_batch_maml): x_train, y_train, x_qry, y_qry = batch_gen.getBatch(qry=False) y_pred, loss = maml(x_train, y_train, x_qry, y_qry) epoch_loss += loss print("Epoch loss: %f" % epoch_loss) print("End of training.") # testing print("Evaluating the maml model") maml.eval() x_test, y_test = feature_list[args.task_num-1][idx_test].to(device), label[idx_test].to(device) auc_roc, auc_pr, ap = maml.evaluating(x_test, y_test) print("End of evaluating.") f.write("MAML auc_roc: %.5f, auc_pr: %.5f, ap: %.5f\n" % (auc_roc, auc_pr, ap)) # g-dev training print('G-dev training...') features, labels, idx_labeled, idx_unlabeled, idx_test = SGC_process(taskData.target, degree=2, l_ratio=lr_s, tr_ratio=tr) # print("finish loading data...") attr_dim = features.shape[1] # print("%d-th run:" % i) # model = FCNet(attr_dim, 1).to(device) # model = SGC_original(attr_dim, 1).to(device) model = SGC(attr_dim, 1).to(device) # print(model) optim = torch.optim.Adam(model.parameters(), lr=0.002, weight_decay=0) # loss = deviation_loss() data_sampler = DataLoader(features, idx_labeled, idx_unlabeled, b_size=8) model.float() model.train() for e in range(1, nb_epochs + 1): # print('Epoch: %d' % e) epoch_loss = 0 epoch_acc = 0 for b in range(nb_batch): x_b, y_b = data_sampler.getBatch() x_b, y_b = x_b.to(device), y_b.to(device) y_pred = model(x_b) loss = deviation_loss(y_b, y_pred) optim.zero_grad() loss.backward() optim.step() epoch_loss += loss.item() print("epoch loss %f" % epoch_loss) # validation model.eval() # print(idx_val.shape) x_val = features[idx_test].to(device) # print(x_val.shape) y_pred = model(x_val).detach().cpu().numpy() y_val = labels[idx_test].detach().cpu().numpy() # fpr, tpr, roc_auc = dict(), dict(), dict() # for i in range(2): # fpr[i], tpr[i], _ = roc_curve(y_val, y_pred, pos_label=1) # roc_auc[i] = auc(fpr[i], tpr[i]) auc_roc, _, auc_pr = aucPerformance(y_val, y_pred) print("G-dev auc_roc: %.5f, auc_pr: %.5f" % (auc_roc, auc_pr)) f.write("G-Dev auc_roc: %.5f, auc_pr: %.5f\n" % (auc_roc, auc_pr)) f.close()
def main(args): config = [('conv2d', [64, 1, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 2, 2, 1, 0]), ('relu', [True]), ('bn', [64]), ('flatten', []), ('linear', [args.n_way, 64])] device = torch.device("cuda" if torch.cuda.is_available() else "cpu") maml = Meta(args, config, device).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print('Total trainable tensors:', num) db_train = OmniglotNShot('./', batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz) for step in range(args.epoch): x_spt, y_spt, x_qry, y_qry = db_train.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).long().to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).long().to(device) # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs = maml(x_spt, y_spt, x_qry, y_qry) print('trainstep:', step, '\ttraining acc:', accs) if (step + 1) % 500 == 0: accs = [] for _ in range(1000 // args.task_num): # test x_spt, y_spt, x_qry, y_qry = db_train.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).long().to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).long().to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip( x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print('Test acc:', accs) ############################## for i in range(args.prune_iteration): # prune print("the {}th prune step".format(i)) x_spt, y_spt, x_qry, y_qry = db_train.getHoleTrain() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).long().to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).long().to(device) maml.prune(x_spt, y_spt, x_qry, y_qry, args.prune_number_one_epoch, args.max_prune_number) # fine-tuning print("start finetuning....") finetune_epoch = args.finetune_epoch finetune_epoch = finetune_epoch * (2 if i == args.prune_iteration - 1 else 1) for step in range(args.finetune_epoch): x_spt, y_spt, x_qry, y_qry = db_train.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).long().to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).long().to(device) accs = maml(x_spt, y_spt, x_qry, y_qry, finetune=True) print('finetune step:', step, '\ttraining acc:', accs) # print the test accuracy after pruning print("start testing....") accs = [] for _ in range(1000 // args.task_num): # test x_spt, y_spt, x_qry, y_qry = db_train.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).long().to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).long().to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip( x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) #np.random.seed(222) config = [('conv2d', [32, 3, 3, 3, 1, 1]), ('bn', [32]), ('relu', [True]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('bn', [32]), ('relu', [True]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('bn', [32]), ('relu', [True]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 1]), ('bn', [32]), ('relu', [True]), ('max_pool2d', [2, 2, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5])] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) root = '/mnt/aitrics_ext/ext01/yanbin/MAML-Pytorch/data/miniImagenet' trainset = MiniImagenet(root, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testset = MiniImagenet(root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) trainloader = DataLoader(trainset, batch_size=args.task_num, shuffle=True, num_workers=4, worker_init_fn=worker_init_fn, drop_last=True) testloader = DataLoader(testset, batch_size=1, shuffle=True, num_workers=1, worker_init_fn=worker_init_fn, drop_last=True) train_data = inf_get(trainloader) test_data = inf_get(testloader) best_acc = 0.0 if not os.path.exists('ckpt/{}'.format(args.exp)): os.mkdir('ckpt/{}'.format(args.exp)) for epoch in range(args.epoch): np.random.seed() x_spt, y_spt, x_qry, y_qry = train_data.__next__() x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if epoch % 100 == 0: print('epoch:', epoch, '\ttraining acc:', accs) if epoch % 2500 == 0: # evaluation # save checkpoint torch.save(maml.state_dict(), 'ckpt/{}/model_{}.pkl'.format(args.exp, epoch)) accs_all_test = [] for _ in range(600): x_spt, y_spt, x_qry, y_qry = test_data.__next__() x_spt, y_spt, x_qry, y_qry = x_spt.squeeze().to( device), y_spt.squeeze().to(device), x_qry.squeeze().to( device), y_qry.squeeze().to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs) with open('ckpt/' + args.exp + '/test.txt', 'a') as f: print('test epoch {}: acc:{:.4f}'.format(epoch, accs[-1]), file=f)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') maml = Meta(args, STSTNet()).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet('/home/lf/miniImagenet/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) mini_test = MiniImagenet('/home/lf/miniImagenet/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.imgsz) for epoch in range(args.epoch // 10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in tqdm(enumerate(db)): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('step:', step, '\ttraining acc:', accs) if step % 500 == 0: # evaluation db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs) torch.save(maml.net, 'maml_ststnet.pth')
def main(): #print(args) TARGET_MODEL = 3 config = [ ('conv2d', [16, 1, 3, 3, 1, 1]), ('relu', [True]), ('bn', [16]), ('conv2d', [32, 16, 4, 4, 2, 1]), ('relu', [True]), ('bn', [32]), ('conv2d', [64, 32, 4, 4, 2, 1]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 4, 4, 2, 1]), ('relu', [True]), ('bn', [64]), ('convt2d', [64, 32, 3, 3, 2, 0]), ('relu', [True]), ('bn', [32]), ('convt2d', [32, 16, 4, 4, 2, 1]), ('relu', [True]), ('bn', [16]), ('convt2d', [16, 8, 4, 4, 2, 1]), ('relu', [True]), ('bn', [8]), ('convt2d', [8, 1, 3, 3, 1, 1]), ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) #print(maml) #print('Total trainable tensors:', num) # initiate different datasets minis = [] for i in range(args.task_num): path = osp.join("./zoo_cw_grad_mnist/train", MODELS[i] + "_mnist.npy") mini = mnist(path, mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.imgsz) db = DataLoader(mini, args.batchsize, shuffle=True, num_workers=0, pin_memory=True) minis.append(db) path_test = osp.join("./zoo_cw_grad_mnist/test", MODELS[TARGET_MODEL] + "_mnist.npy") mini_test = mnist(path_test, mode='test', n_way=1, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.imgsz) mini_test = DataLoader(mini_test, 10, shuffle=True, num_workers=0, pin_memory=True) # start training step_number = len(minis[0]) test_step_number = len(mini_test) BEST_ACC = 1.0 target_model = get_target_model(TARGET_MODEL).to(device) def save_model(model,acc): model_file_path = './checkpoint/mnist' if not os.path.exists(model_file_path): os.makedirs(model_file_path) file_name = str(acc) + 'mnist_'+ MODELS[TARGET_MODEL] + '.pt' save_model_path = os.path.join(model_file_path, file_name) torch.save(model.state_dict(), save_model_path) def load_model(model,acc): model_checkpoint_path = './checkpoint/mnist/' + str(acc) + 'mnist_' + MODELS[TARGET_MODEL] + '.pt' assert os.path.exists(model_checkpoint_path) model.load_state_dict(torch.load(model_checkpoint_path)) return model for epoch in range(args.epoch//100): minis_iter = [] for i in range(len(minis)): minis_iter.append(iter(minis[i])) mini_test_iter = iter(mini_test) if args.resume: maml = load_model(maml,"0.7231071") for step in range(step_number): batch_data = [] for each in minis_iter: batch_data.append(each.next()) accs = maml(batch_data, device) if (step + 1) % step_number == 0: print('Training acc:', accs) if accs[0] < BEST_ACC: BEST_ACC = accs[0] save_model(maml, BEST_ACC) if (epoch + 1) % 15 == 0 and step ==0: # evaluation accs_all_test = [] for i in range(3): test_data = mini_test_iter.next() accs = maml.finetunning(test_data, target_model, device) accs_all_test.append(accs) accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): # Manually seed torch and numpy for reproducible results torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) np.random.seed(args.seed) # Open csv file to write for metric logging try: f = open("results.csv", "w") except FileNotFoundError: f = open("results.csv", "x") f.write("Steps,tr_loss,tr_acc,val_loss,val_acc,te_loss,te_acc\n") # Choose PyTorch device and create the model device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') model = Meta(args, defineModel(args)).to(device) # Setup Weights and Biases logger, config hyperparams and watch model wandb.init(project="Meta-SGD") name = f"N{args.n_way}K{args.k_spt}" if args.update_step > 0: name += "Meta" name += f"Ret{args.ret_channels}VVS{args.vvs_depth}KS{args.kernel_size}" wandb.run.name = name wandb.config.update(args) wandb.watch(model) print(f"RUN NAME: {name}") # Print additional information on the model if args.verbose: tmp = filter(lambda x: x.requires_grad, model.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(args) print(model) print('Total trainable tensors:', num) # Create datasets # batchsz here means total episode number print("\nGathering Datasets:") mini = MiniImagenet('miniimagenet/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.imgsz) mini_train_eval = MiniImagenet('miniimagenet/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.eval_steps, resize=args.imgsz) mini_val = MiniImagenet('miniimagenet/', mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.eval_steps, resize=args.imgsz) mini_test = MiniImagenet('miniimagenet/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.eval_steps, resize=args.imgsz) print("\nMeta-Training:") pruning_factor = args.pruning best_tr_acc, best_val_acc, best_te_acc = 0, 0, 0 epoch_bar = tqdm(range(args.epoch // 10000), desc="Training", total=len(range(args.epoch // 10000))) for epoch in epoch_bar: # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) task_bar = tqdm(enumerate(db), desc=f"Epoch {epoch}", total=len(db), leave=False) for step, (x_spt, y_spt, x_qry, y_qry) in task_bar: total_steps = len(db) * epoch + step + 1 # Perform training for each task x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) model(x_spt, y_spt, x_qry, y_qry) if total_steps % args.save_summary_steps == 0: # evaluation # Get evaluation metrics tr_acc, tr_loss = evaluate(model, mini_train_eval, device, "Eval Train") val_acc, val_loss = evaluate(model, mini_val, device, "Eval Val") te_acc, te_loss = evaluate(model, mini_test, device) # Update Task tqdm bar metrics = { 'tr acc': tr_acc, 'val_acc': val_acc, 'te_acc': te_acc } task_bar.set_postfix(metrics) metrics['tr_loss'] = tr_loss metrics['val_loss'] = val_loss metrics['te_loss'] = te_loss wandb.log(metrics) f.write( f"{total_steps},{tr_loss},{tr_acc},{val_loss},{val_acc},{te_loss},{te_acc}\n" ) # Update best metrics if val_acc > best_val_acc: best_val_acc = val_acc pruning_factor = args.pruning else: pruning_factor -= 1 best_te_acc = max(te_acc, best_te_acc) best_tr_acc = max(tr_acc, best_tr_acc) if pruning_factor == 0: break # Update tqdm epoch_bar.set_postfix({ 'b_tr_acc': best_tr_acc, 'b_val_acc': best_val_acc, 'b_te_acc': best_te_acc, 'prune': pruning_factor }) if pruning_factor == 0: break f.close()
def main(): mem_usage = memory_usage(-1, interval=.5, timeout=1) torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) root = args.data_dir feat = np.load(root + 'features.npy', allow_pickle=True) with open(root + '/graph_dgl.pkl', 'rb') as f: dgl_graph = pickle.load(f) if args.task_setup == 'Disjoint': with open(root + 'label.pkl', 'rb') as f: info = pickle.load(f) elif args.task_setup == 'Shared': if args.task_mode == 'True': root = root + '/task' + str(args.task_n) + '/' with open(root + 'label.pkl', 'rb') as f: info = pickle.load(f) total_class = len(np.unique(np.array(list(info.values())))) print('There are {} classes '.format(total_class)) if args.task_setup == 'Disjoint': labels_num = args.n_way elif args.task_setup == 'Shared': labels_num = total_class if len(feat.shape) == 2: # single graph, to make it compatible to multiple graph retrieval. feat = [feat] config = [('GraphConv', [feat[0].shape[1], args.hidden_dim])] if args.h > 1: config = config + [('GraphConv', [args.hidden_dim, args.hidden_dim]) ] * (args.h - 1) config = config + [('Linear', [args.hidden_dim, labels_num])] if args.link_pred_mode == 'True': config.append(('LinkPred', [True])) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) max_acc = 0 model_max = copy.deepcopy(maml) db_train = Subgraphs(root, 'train', info, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=args.batchsz, args=args, adjs=dgl_graph, h=args.h) db_val = Subgraphs(root, 'val', info, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, args=args, adjs=dgl_graph, h=args.h) db_test = Subgraphs(root, 'test', info, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, args=args, adjs=dgl_graph, h=args.h) print('------ Start Training ------') s_start = time.time() max_memory = 0 for epoch in range(args.epoch): db = DataLoader(db_train, args.task_num, shuffle=True, num_workers=args.num_workers, pin_memory=True, collate_fn=collate) s_f = time.time() for step, (x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry) in enumerate(db): nodes_len = 0 if step >= 1: data_loading_time = time.time() - s_r else: data_loading_time = time.time() - s_f s = time.time() # x_spt: a list of #task_num tasks, where each task is a mini-batch of k-shot * n_way subgraphs # y_spt: a list of #task_num lists of labels. Each list is of length k-shot * n_way int. nodes_len += sum([sum([len(j) for j in i]) for i in n_spt]) accs = maml(x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry, feat) max_memory = max(max_memory, float(psutil.virtual_memory().used / (1024**3))) if step % args.train_result_report_steps == 0: print('Epoch:', epoch + 1, ' Step:', step, ' training acc:', str(accs[-1])[:5], ' time elapsed:', str(time.time() - s)[:5], ' data loading takes:', str(data_loading_time)[:5], ' Memory usage:', str(float(psutil.virtual_memory().used / (1024**3)))[:5]) s_r = time.time() # validation per epoch db_v = DataLoader(db_val, 1, shuffle=True, num_workers=args.num_workers, pin_memory=True, collate_fn=collate) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry in db_v: accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry, feat) accs_all_test.append(accs) accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Epoch:', epoch + 1, ' Val acc:', str(accs[-1])[:5]) if accs[-1] > max_acc: max_acc = accs[-1] model_max = copy.deepcopy(maml) db_t = DataLoader(db_test, 1, shuffle=True, num_workers=args.num_workers, pin_memory=True, collate_fn=collate) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry in db_t: accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry, feat) accs_all_test.append(accs) accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', str(accs[1])[:5]) for x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry in db_t: accs = model_max.finetunning(x_spt, y_spt, x_qry, y_qry, c_spt, c_qry, n_spt, n_qry, g_spt, g_qry, feat) accs_all_test.append(accs) #torch.save(model_max.state_dict(), './model.pt') accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Early Stopped Test acc:', str(accs[-1])[:5]) print('Total Time:', str(time.time() - s_start)[:5]) print('Max Momory:', str(max_memory)[:5])
def main(args): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) device = torch.device('cuda') config = [] if args.arch == "Unet": for block in range(args.NUM_DOWN_CONV): out_channels = (2**block) * args.HIDDEN_DIM if (block == 0): config += [( 'conv2d', [out_channels, args.imgc, 3, 3, 1, 1] ) # out_c, in_c, k_h, k_w, stride, padding, also only conv, without bias ] else: config += [ ('conv2d', [out_channels, out_channels // 2, 3, 3, 1, 1]), # out_c, in_c, k_h, k_w, stride, padding ] config += [ ('leakyrelu', [0.2, False]), # alpha; if true then executes relu in place ('bn', [out_channels]) ] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('max_pool2d', [2, 2, 0])] # kernel_size, stride, padding for block in range(args.NUM_DOWN_CONV - 1): out_channels = (2**(args.NUM_DOWN_CONV - block - 2)) * args.HIDDEN_DIM in_channels = out_channels * 3 config += [('upsample', [2])] config += [('conv2d', [out_channels, in_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [('conv2d', [out_channels, out_channels, 3, 3, 1, 1]), ('leakyrelu', [0.2, False]), ('bn', [out_channels])] config += [ ('conv2d_b', [args.outc, args.HIDDEN_DIM, 3, 3, 1, 1]) ] # all the conv2d before are without bias, and this conv_b is with bias else: raise ("architectures other than Unet hasn't been added!!") maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) # print(maml) for name, param in maml.named_parameters(): print(name, param.size()) print('Total trainable tensors:', num) SUMMARY_INTERVAL = 5 TEST_PRINT_INTERVAL = SUMMARY_INTERVAL * 5 ITER_SAVE_INTERVAL = 300 EPOCH_SAVE_INTERVAL = 5 model_path = "/scratch/users/chenkaim/pytorch-models/pytorch_" + args.model_name + "_k_shot_" + str( args.k_spt) + "_task_num_" + str(args.task_num) + "_meta_lr_" + str( args.meta_lr) + "_inner_lr_" + str( args.update_lr) + "_num_inner_updates_" + str(args.update_step) if not os.path.isdir(model_path): os.mkdir(model_path) start_epoch = 0 if (args.continue_train): print("Restoring weights from ", model_path + "/epoch_" + str(args.continue_epoch) + ".pt") checkpoint = torch.load(model_path + "/epoch_" + str(args.continue_epoch) + ".pt") maml = checkpoint['model'] maml.lr_scheduler = checkpoint['lr_scheduler'] maml.meta_optim = checkpoint['optimizer'] start_epoch = args.continue_epoch db = RCWA_data_loader(batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz, data_folder=args.data_folder) for step in range(start_epoch, args.epoch): print("epoch: ", step) if step % EPOCH_SAVE_INTERVAL == 0: checkpoint = { 'epoch': step, 'model': maml, 'optimizer': maml.meta_optim, 'lr_scheduler': maml.lr_scheduler } torch.save(checkpoint, model_path + "/epoch_" + str(step) + ".pt") for itr in range( int(0.7 * db.total_data_samples / ((args.k_spt + args.k_qry) * args.task_num))): x_spt, y_spt, x_qry, y_qry = db.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs, loss_q = maml(x_spt, y_spt, x_qry, y_qry) if itr % SUMMARY_INTERVAL == 0: print_str = "Iteration %d: pre-inner-loop train accuracy: %.5f, post-iner-loop test accuracy: %.5f, train_loss: %.5f" % ( itr, accs[0], accs[-1], loss_q) print(print_str) if itr % TEST_PRINT_INTERVAL == 0: accs = [] for _ in range(10): # test x_spt, y_spt, x_qry, y_qry = db.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip( x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print( 'Meta-validation pre-inner-loop train accuracy: %.5f, meta-validation post-inner-loop test accuracy: %.5f' % (accs[0], accs[-1])) maml.lr_scheduler.step()
def main(args): if not os.path.exists('./logs'): os.mkdir('./logs') logfile = os.path.sep.join(('.', 'logs', f'omniglot_way[{args.n_way}]_shot[{args.k_spt}].json')) if args.write_log: log_fp = open(logfile, 'wb') torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [ ('conv2d', [64, 1, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 3, 3, 2, 0]), ('relu', [True]), ('bn', [64]), ('conv2d', [64, 64, 2, 2, 1, 0]), ('relu', [True]), ('bn', [64]), ('flatten', []), ('linear', [args.n_way, 64]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) path = os.sep.join((os.path.dirname(__file__), 'dataset', 'omniglot')) db_train = OmniglotNShot(path, batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz) for step in range(args.epoch): # 获取一定的 epoch 数据. 在omniglot NShot类里写的是 x_spt, y_spt, x_qry, y_qry = db_train.next() x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 50 == 0: print('step:', step, '\ttraining acc:', accs) if step % 500 == 0: accs = [] for _ in range(1000 // args.task_num): # test x_spt, y_spt, x_qry, y_qry = db_train.next('test') x_spt, y_spt, x_qry, y_qry = torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), \ torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip(x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): torch.manual_seed(222) # 为cpu设置种子,为了使结果是确定的 torch.cuda.manual_seed_all(222) # 为GPU设置种子,为了使结果是确定的 np.random.seed(222) print(args) config = [ ('conv2d', [32, 1, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 7040]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet("./miniimagenet", mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000) mini_test = MiniImagenet("./miniimagenet", mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100) last_accuracy = 0 plt_train_loss = [] plt_train_acc = [] plt_test_loss = [] plt_test_acc =[] for epoch in range(args.epoch // 10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(device), x_qry.to(device), y_qry.to(device) accs, loss_q = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: d = loss_q.cpu() dd = d.detach().numpy() plt_train_loss.append(dd) plt_train_acc.append(accs[-1]) print('step:', step, '\ttraining acc:', accs) if step % 50 == 0: # evaluation db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] loss_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs, loss_test= maml.finetunning(x_spt, y_spt, x_qry, y_qry) loss_all_test.append(loss_test) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) plt_test_acc.append(accs[-1]) avg_loss = np.mean(np.array(loss_all_test)) plt_test_loss.append(avg_loss) print('Test acc:', accs) test_accuracy = np.mean(np.array(accs)) if test_accuracy > last_accuracy: # save networks torch.save(maml.state_dict(), str( "./models/miniimagenet_maml" + str(args.n_way) + "way_" + str( args.k_spt) + "shot.pkl")) last_accuracy = test_accuracy plt.figure() plt.title("testing info") plt.xlabel("episode") plt.ylabel("Acc/loss") plt.plot(plt_test_loss, label='Loss') plt.plot(plt_test_acc, label='Acc') plt.legend(loc='upper right') plt.savefig('./drawing/test.png') plt.show() plt.figure() plt.title("training info") plt.xlabel("episode") plt.ylabel("Acc/loss") plt.plot(plt_train_loss, label='Loss') plt.plot(plt_train_acc, label='Acc') plt.legend(loc='upper right') plt.savefig('./drawing/train.png') plt.show()
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) print(args) config = [('conv2d', [32, 3, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 32 * 5 * 5])] device = torch.device("cuda" if torch.cuda.is_available() else "cpu") maml = Meta(args, config).to(device) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet('./data/', mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000, resize=args.img_sz) mini_test = MiniImagenet('./data/', mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100, resize=args.img_sz) for epoch in range(args.epoch // 10000): # fetch meta_batchsz num of episode each time db = DataLoader(mini, args.task_num, shuffle=True, num_workers=1, pin_memory=True) for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(db): x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to( device), x_qry.to(device), y_qry.to(device) accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 30 == 0: print('step:', step, '\ttraining acc:', accs) if step % 500 == 0: # evaluation db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs)
def main(): torch.manual_seed(222) torch.cuda.manual_seed_all(222) #np.random.seed(222) test_result = {} best_acc = 0.0 maml = Meta(args, Param.config).to(Param.device) if len(args.gpu.split(',')) > 1: maml = torch.nn.DataParallel(maml) state_dict = torch.load(Param.out_path + args.ckpt) print(state_dict.keys()) pretrained_dict = OrderedDict() for k in state_dict.keys(): if n_gpus == -1: pretrained_dict[k[7:]] = deepcopy(state_dict[k]) else: pretrained_dict[k[0:]] = deepcopy(state_dict[k]) maml.load_state_dict(pretrained_dict) print("Load from ckpt:", Param.out_path + args.ckpt) #opt = optim.Adam(maml.parameters(), lr=args.meta_lr) #opt = optim.SGD(maml.parameters(), lr=args.meta_lr, momentum=0.9, weight_decay=args.weight_decay) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) if args.loader in [0, 1]: # default loader if args.loader == 1: #from dataloader.mini_imagenet import MiniImageNet as MiniImagenet from MiniImagenet2 import MiniImagenet else: from MiniImagenet import MiniImagenet testset = MiniImagenet(Param.root, mode='test', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, resize=args.imgsz) testloader = DataLoader(testset, batch_size=1, shuffle=False, num_workers=4, worker_init_fn=worker_init_fn, drop_last=True) test_data = inf_get(testloader) elif args.loader == 2: # pkl loader args_data = {} args_data['x_dim'] = "84,84,3" args_data['ratio'] = 1.0 args_data['seed'] = 222 loader_test = dataset_mini(600, 100, 'test', args_data) loader_test.load_data_pkl() """Test for 600 epochs (each has 4 tasks)""" ans = None maml_clone = deepcopy(maml) for itr in range(600): # 600x4 test tasks if args.loader in [0, 1]: support_x, support_y, qx, qy = test_data.__next__() support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) elif args.loader == 2: support_x, support_y, qx, qy = get_data(loader_test) support_x, support_y, qx, qy = support_x.to( Param.device), support_y.to(Param.device), qx.to( Param.device), qy.to(Param.device) temp = maml_clone(support_x, support_y, qx, qy, meta_train=False) if (ans is None): ans = temp else: ans = torch.cat([ans, temp], dim=0) if itr % 100 == 0: print(itr, ans.mean(dim=0).tolist()) meanacc = np.array(ans.mean(dim=0).tolist()) stdacc = np.array(ans.std(dim=0).tolist()) ci95 = 1.96 * stdacc / np.sqrt(600) print(f'Acc: {meanacc[-1]:.4f}, ci95: {ci95[-1]:.4f}') with open(Param.out_path + 'test.txt', 'w') as f: print(f'Acc: {meanacc[-1]:.4f}, ci95: {ci95[-1]:.4f}', file=f)
def main(): torch.manual_seed(222) # 为cpu设置种子,为了使结果是确定的 torch.cuda.manual_seed_all(222) # 为GPU设置种子,为了使结果是确定的 np.random.seed(222) print(args) config = [ ('conv2d', [32, 1, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 2, 0]), ('conv2d', [32, 32, 3, 3, 1, 0]), ('relu', [True]), ('bn', [32]), ('max_pool2d', [2, 1, 0]), ('flatten', []), ('linear', [args.n_way, 7040]) ] device = torch.device('cuda') maml = Meta(args, config).to(device) if os.path.exists( "./models/" + str("./models/miniimagenet_maml" + str(args.n_way) + "way_" + str(args.k_spt) + "shot.pkl")): path = "./models/" + str("./models/miniimagenet_maml" + str(args.n_way) + "way_" + str(args.k_spt) + "shot.pkl") maml.load_state_dict(path) print("load model success") tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print('Total trainable tensors:', num) # batchsz here means total episode number mini = MiniImagenet("./miniimagenet", mode='train', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=10000) mini_test = MiniImagenet("./miniimagenet", mode='val', n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, batchsz=100) test_accuracy = [] for epoch in range(10): # fetch meta_batchsz num of episode each time db_test = DataLoader(mini_test, 1, shuffle=True, num_workers=1, pin_memory=True) accs_all_test = [] for x_spt, y_spt, x_qry, y_qry in db_test: x_spt, y_spt, x_qry, y_qry = x_spt.squeeze(0).to(device), y_spt.squeeze(0).to(device), \ x_qry.squeeze(0).to(device), y_qry.squeeze(0).to(device) accs, loss_t = maml.finetunning(x_spt, y_spt, x_qry, y_qry) accs_all_test.append(accs) # [b, update_step+1] accs = np.array(accs_all_test).mean(axis=0).astype(np.float16) print('Test acc:', accs) test_accuracy.append(accs[-1]) average_accuracy = sum(test_accuracy) / len(test_accuracy) print("average accuracy:{}".format(average_accuracy))
def main(args): torch.manual_seed(222) torch.cuda.manual_seed_all(222) np.random.seed(222) torch.backends.cudnn.benchmark=True print(args) config = [ ("conv2d", [64, 1, 3, 3, 2, 0]), ("relu", [True]), ("bn", [64]), ("conv2d", [64, 64, 3, 3, 2, 0]), ("relu", [True]), ("bn", [64]), ("conv2d", [64, 64, 3, 3, 2, 0]), ("relu", [True]), ("bn", [64]), ("conv2d", [64, 64, 2, 2, 1, 0]), ("relu", [True]), ("bn", [64]), ("flatten", []), ("linear", [args.n_way, 64]), ] device = torch.device("cuda") maml = Meta(args, config).to(device) db_train = OmniglotNShot( "omniglot", batchsz=args.task_num, n_way=args.n_way, k_shot=args.k_spt, k_query=args.k_qry, imgsz=args.imgsz, ) tmp = filter(lambda x: x.requires_grad, maml.parameters()) num = sum(map(lambda x: np.prod(x.shape), tmp)) print(maml) print("Total trainable tensors:", num) for step in range(args.epoch): x_spt, y_spt, x_qry, y_qry = db_train.next() x_spt, y_spt, x_qry, y_qry = ( torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device), ) # x_spt: shape: 32, 5, 1, 28, 28 # y_spt: shape: 32, 5 # x_qry: 32, 75, 1, 28, 28 # y_qry: 32, 75 # set traning=True to update running_mean, running_variance, bn_weights, bn_bias accs = maml(x_spt, y_spt, x_qry, y_qry) if step % 50 == 0: print("step:", step, "\ttraining acc:", accs) if step % 500 == 0: accs = [] for _ in range(1000 // args.task_num): # test x_spt, y_spt, x_qry, y_qry = db_train.next("test") x_spt, y_spt, x_qry, y_qry = ( torch.from_numpy(x_spt).to(device), torch.from_numpy(y_spt).to(device), torch.from_numpy(x_qry).to(device), torch.from_numpy(y_qry).to(device), ) # split to single task each time for x_spt_one, y_spt_one, x_qry_one, y_qry_one in zip(x_spt, y_spt, x_qry, y_qry): test_acc = maml.finetunning(x_spt_one, y_spt_one, x_qry_one, y_qry_one) accs.append(test_acc) # [b, update_step+1] accs = np.array(accs).mean(axis=0).astype(np.float16) print("Test acc:", accs)