def get_results_advtr(args, ds_src, ds_tar): from learning import LearnerDACls from learning import TempScalingCls as CalibratorCls exp_name_list = glob.glob('snapshots/m2m_advtr_small_*') error_cls_list, error_cal_list = [], [] for exp_name in exp_name_list: params = types.SimpleNamespace() params.save_root = exp_name params.find_best = False params.load_final = False params.advtr_type = 'DANN' params.schedule_reg_param_adv = True print(params.save_root) ## init a base model mdl = getattr(model, args.model.base)(num_class=args.data.n_labels, input_shape=args.data.img_size) ## init an adv model mdl_adv = getattr(model, 'BigAdvFNN')(n_in=mdl.dim_feat) ## init a learner learner = LearnerDACls(params, model.DAN(mdl, mdl_adv)) ## train the model learner.train([ds_src.train, ds_dom.train], None, None) ## test the model error_cls, error_cal, *_ = learner.test(ds_tar.test, ld_name=args.data.tar, verbose=True) error_cls_list.append(error_cls.numpy()) error_cal_list.append(error_cal) return np.array(error_cls_list), np.array(error_cal_list)
def _train_begin(self, ds_src, ds_tar, ds_dom): """A function called in the begning of training process. It trains an initial model for self-training. ds_src: source dataset loader, which contains loaders for train/val/test ds_tar: target dataset loader, which contains loaders for train/val/test ds_dom: domain dataset loader, which contains loaders for train/val/test """ ## initialize a base model if self.params.init_advtr: print("## init the teacher model with adversarial training") model_base = self.model model.set_trainable(model_base, True) ## init a adv model mdl_adv = getattr( model, self.params_init.model_advtr)(n_in=model_base.dim_feat) ## init a learner learner = LearnerDACls(self.params_init, model.DAN(model_base, mdl_adv), model_name_postfix='_advtrinit') ## train the model learner.train([ds_src.train, ds_dom.train], None, ds_tar.test) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) print() else: print("## init the teacher model with sourceonly training") model_base = self.model model.set_trainable(model_base, True) ## init a learner learner = LearnerCls(self.params_init, model_base, model_name_postfix='_sourceonlyinit') ## train the model learner.train(ds_src.train, ds_src.val, ds_tar.test) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) print() print('## initialize a pseudo-labeling function') model_base = self.model model_conf = self.model_s.train.model_conf ## init a learner learner = LearnerConfPred(self.params_conf, model_conf, model_base, None, model_name_postfix='_confpredinit') ## set a constant model_conf.T = tf.Variable(1.0 - self.params_conf.eps) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) print()
def main(args): data_fn = 'plots/prec_cov_list.pk' fig_fn = 'plots/prec_cov' if os.path.exists(data_fn): pc_data = pickle.load(open(data_fn, 'rb')) plot_prec_cov(pc_data['T_list'], pc_data['prec_list'], pc_data['cov_list'], fig_fn) return ## init a snapshot path os.makedirs(args.train.save_root, exist_ok=True) ## init logger sys.stdout = Logger(os.path.join(args.train.save_root, 'out')) ## print args print_args(args) ## init gpus if not args.cpu: print("##GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) print() ## init datasets print("## init datasets") ds_src = data.MultiSourceDataset( args.data.src, args.aug_params, batch_size=args.data.batch_size, val_shuffle=True, val_aug=True, domain_id=1, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0]) assert (len(args.aug_params) == 1) ##TODO ds_tar = getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params[0], val_shuffle=True, val_aug=True, domain_id=0, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1]) ds_dom = data.DomainDataset( data.MultiSourceDataset( args.data.src, args.aug_params, batch_size=args.data.batch_size, val_shuffle=True, val_aug=True, test_aug=True, #diff domain_id=1, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0]), getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params[0], val_shuffle=True, val_aug=True, test_aug=True, #diff domain_id=0, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1])) print() #### ## reliable teacher learning #### mdl_st_base = getattr(model, args.model.base)(num_class=args.model.n_labels, input_shape=args.model.img_size) #mdl_st_base = model.TempCls(mdl_st_base) mdl_st = model.Student(args.model, mdl_st_base, ds_src, ds_tar, ideal=args.ideal) mdl_tc_base = getattr(model, args.model.base)(num_class=args.model.n_labels, input_shape=args.model.img_size) #mdl_tc_base = model.TempCls(mdl_tc_base) mdl_tc = model.Teacher(args.model, mdl_tc_base, ds_src, ds_tar, ideal=args.ideal) ## rename model_t = mdl_tc model_s = mdl_st model_c = model_s.model_base params = args.train params_base = args.train_base params_advtr = args.train_advtr params_iw = args.train_iw params_iw_cal = args.cal_iw params_conf = args.est_conf i_epoch = 1 ## init a model if params.init == 'sourceonly': ##TODO: assume classification print("## init the student model with sourceonly training") model.set_trainable(model_c, True) ## init a learner learner = LearnerCls(params_base, model_c, model_name_postfix='_sourceonlyinit') ## train the model learner.train(ds_src.train, ds_src.val, ds_src.test) ## test the model learner.test(ds_src.test, ld_name='src', verbose=True) print() elif params.init == 'advtr': ##TODO: assume classification print("## init a base model with adversarial training") model.set_trainable(model_c, True) ## init a adv model mdl_adv = getattr(model, params_advtr.model_advtr)(n_in=model_c.dim_feat) ## init a learner learner = LearnerDACls(params_advtr, model.DAN(model_c, mdl_adv), model_name_postfix='_advtrinit') ## train the model learner.train([ds_src.train, ds_dom.train], None, ds_tar.test) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) print() else: raise NotImplementedError ## init iw if model_t.train.model_conf.model_iw is not None: print("## learn IW") model_sd = model_t.train.model_conf.model_iw.model_sd.model model_sd.train() ## init a learner learner_sd = LearnerCls(params_iw, model_sd, model_name_postfix='_iw_epoch_%d' % (i_epoch)) ## train the model learner_sd.train(ds_dom.train, ds_dom.val, ds_dom.test) ## test the model learner_sd.test(ds_dom.test, ld_name='domain', verbose=True) print() ## init a calibraton model model_sd_cal = model_t.train.model_conf.model_iw.model_sd model_sd_cal.train() ## init a calibrator calibrator_iw = CalibratorCls(params_iw_cal, model_sd_cal, model_name_postfix='_iw_cal_epoch_%d' % (i_epoch)) ## calibrate the model calibrator_iw.train(ds_dom.val, ds_dom.val, ds_dom.test) ## test the model calibrator_iw.test(ds_dom.test, ld_name='domain', verbose=True) print() ## 2. learn confidence predictor model_base = model_t.train.model_base model_conf = model_t.train.model_conf model_iw = model_t.train.model_conf.model_iw model_iw_cond = model.CondIW(model_iw, model_conf, ds_src.train, ds_tar.train) ## init a learner learner = LearnerConfPred(params_conf, model_conf, model_base, model_iw_cond, model_name_postfix='_confpred_epoch_%d' % (i_epoch)) ## train the model learner.train(ds_src.val, ds_src.val, ds_tar.test) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) learner.test(ds_tar.train, ld_name='tar (train)', verbose=True) print() else: model_base = model_t.train.model_base model_conf = model_t.train.model_conf ## init a learner learner = LearnerConfPred(params_conf, model_conf, model_base, None, model_name_postfix='_confpred_epoch_%d' % (i_epoch)) ## train the model model_conf.T = tf.Variable(1.0 - params_conf.eps) ##TODO print("T = %f" % (model_conf.T.numpy())) ## test the model learner.test(ds_tar.test, ld_name='tar', verbose=True) learner.test(ds_tar.train, ld_name='tar (train)', verbose=True) ## compute precision and coverage T_list, prec_list, cov_list = [], [], [] for T in np.arange(0.0, 1.0, 0.01): model_conf.T = T prec, n_conf, n = learner.test(ds_tar.train, ld_name='tar (train)', verbose=True) T_list.append(T) prec_list.append(prec.numpy()) cov_list.append(float(n_conf) / float(n)) print(T_list) print(prec_list) print(cov_list) print() T_list = np.array(T_list) prec_list = np.array(prec_list) cov_list = np.array(cov_list) pickle.dump( { 'T_list': T_list, 'prec_list': prec_list, 'cov_list': cov_list }, open(data_fn, 'wb'))
## init datasets print("## init datasets") # ds_src = data.MultiSourceDataset(args.src, [None], batch_size=args.batch_size, val_shuffle=True, val_aug=True, domain_id=1, # color=False if args.img_size[2]==1 else True, size=args.img_size[0]) ds_tar = getattr(data, args.tar)(root=os.path.join('data', args.tar.lower()), batch_size=args.batch_size, val_shuffle=True, domain_id=0, color=False if args.img_size[2]==1 else True, size=args.img_size[0]) ## gather results error_list, ece_list, ent_list = [], [], [] for reg in reg_param_adv_list: ## init a base model mdl_base = getattr(model, args.model)(num_class=args.n_labels, activation='relu', input_shape=args.img_size) ##TODO: generalize mdl_adv = getattr(model, args.model_adv)(n_in=800)#mdl_base.dim_feat) mdl = model.DAN(mdl_base, mdl_adv) if args.best_model: mdl.load_weights('snapshots/%s2%s_advtr_%s%s%s_%s/model_params_best'%(src_short[0].lower(), args.tar[0].lower(), reg.replace('.', 'd'), '_smalln' if args.smalln else '', '_aug' if args.aug else '', args.model)) else: mdl.load_weights('snapshots/%s2%s_advtr_%s%s%s_%s/model_params_final'%(src_short[0].lower(), args.tar[0].lower(), reg.replace('.', 'd'), '_smalln' if args.smalln else '', '_aug' if args.aug else '', args.model)) ## compute error and ece learner = LearnerDACls(None, mdl) error, ece, ece_oc, ent = learner.test(ds_tar.test, ld_name=args.tar) error_list.append(error.numpy())
dl_trn, vocab, lenc, ldec = dataset.load(batchSize=args.batch_size, seqLen=args.seq_len, \ path=DATA_PATH, cl=args.clip, voc=None, lenc=None, ldec=None, bertToks=args.bert_tokens) dl_val, val_vocab, vlenc, vldec = dataset.load(batchSize =args.batch_size, seqLen = args.seq_len, \ path = VALID_PATH, cl=args.clip, voc=vocab, lenc=lenc, ldec=ldec, bertToks=args.bert_tokens) dl_tst, tst_vocab, tlenc, tdec = dataset.load(batchSize = 1, seqLen = args.seq_len, \ path = TEST_PATH, cl=args.clip, voc=vocab, lenc=lenc, ldec=ldec, bertToks=args.bert_tokens) device = torch.device( "cuda" if torch.cuda.is_available() and args.cuda else "cpu") modeldict = { "bert": model.BertMLP(n_classes = len(lenc)), "cnn": model.CNN(n_classes = len(lenc), vocab_size = len(vocab), emb_dim = args.embedding_dim, \ n_hidden_units = args.embedding_dim, n_filters = 100, filter_sizes = [3,4,5], device=device), "dan": model.DAN(n_classes = len(lenc), vocab_size = len(vocab), emb_dim = args.embedding_dim, \ n_hidden_units = args.embedding_dim, device=device) } mdl = modeldict[args.model].to(device) criterion = torch.nn.CrossEntropyLoss().to(device) optimizer = torch.optim.Adam(mdl.parameters(), lr=args.lr) json_obj = {"name": args.data_name} best_loss = np.inf for epoch in range(1, args.epochs + 1): loss, json_loss = train(epoch, device) json_obj["epoch {}".format(epoch)] = json_loss if loss < best_loss: best_loss = loss print('* Saved') torch.save(mdl.state_dict(), MODEL_SAV)
def main(args): # ## init a snapshot path # os.makedirs(args.train.save_root, exist_ok=True) # ## init logger # sys.stdout = Logger(os.path.join(args.train.save_root, 'out')) # ## print args # print_args(args) snap_list = glob.glob(args.snapshot_prefix + '_*') print(snap_list) print("# experiments = ", len(snap_list)) ## init gpus if not args.cpu: print("##GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) print() ## init datasets print("## init datasets") ds_src = data.MultiSourceDataset( args.data.src, args.aug_params, batch_size=args.data.batch_size, train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0], resize_pad=True if len(args.data.src) == 1 and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else False, ##TODO: check if it's necessary ) assert (len(args.aug_params) == 1) ##TODO ds_tar = getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params[0], train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1]) ds_dom = data.DomainDataset( data.MultiSourceDataset( args.data.src, args.aug_params, batch_size=args.data.batch_size, train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, test_aug=True, # augment all splits domain_id=1, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0], resize_pad=True if len(args.data.src) == 1 and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else False, ##TODO: check if it's necessary ), getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params[0], train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, test_aug=True, # augment all splits domain_id=0, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1])) ##TODO: redundant ds_src_init = data.MultiSourceDataset( args.data.src, args.aug_params_init, batch_size=args.data.batch_size, train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], ##TODO sample_ratio=args.data.sample_ratio[0], resize_pad=True if len(args.data.src) == 1 and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else False, ##TODO: check if it's necessary ) assert (len(args.aug_params) == 1) ##TODO ds_tar_init = getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params_init[0], ##TODO train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1]) ds_dom_init = data.DomainDataset( data.MultiSourceDataset( args.data.src, args.aug_params_init, batch_size=args.data.batch_size, train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, test_aug=True, # augment all splits domain_id=1, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0], resize_pad=True if len(args.data.src) == 1 and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else False, ##TODO: check if it's necessary ), getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params_init[0], ##TODO train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, test_aug=True, # augment all splits domain_id=0, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1])) ##TODO: redundant ds_src_self = data.MultiSourceDataset( args.data.src, args.aug_params, batch_size=args.data.batch_size, train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, domain_id=1, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[0], resize_pad=True if len(args.data.src) == 1 and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else False, ##TODO: check if... ) assert (len(args.aug_params) == 1) ##TODO ds_tar_self = getattr(data, args.data.tar)( root=os.path.join('data', args.data.tar.lower()), batch_size=args.data.batch_size, aug_list=args.aug_params[0], train_shuffle=True, train_aug=True, val_shuffle=True, val_aug=True, domain_id=0, color=False if args.data.img_size[2] == 1 else True, size=args.data.img_size[0], sample_ratio=args.data.sample_ratio[1], double_aug=True if args.training_type == 'selfcon' else False, ) print() if args.merge_train_val: ds_src.train = data.ChainLoader(ds_src.train, ds_src.val) ds_dom.train = data.ChainLoader(ds_dom.train, ds_dom.val) ## collect stats cls_error_init_list, cal_error_init_list = [], [] cls_error_list, cal_error_list = [], [] perf_epoch_list = [] for snap_root in snap_list: ## ## final student ## ## a student model mdl_st_base = getattr(model, args.model.base)(num_class=args.model.n_labels, input_shape=args.model.img_size) mdl_st = model.Student(args.model, mdl_st_base, ds_src_self, ds_tar_self, ideal=args.ideal) ## load the final student mdl_st.model_base.load_weights( os.path.join(snap_root, 'model_params_final')) ## evaluate learner = LearnerClsSelf(None, None, mdl_st, None) error, ece, *_ = learner.test(ds_tar.test, ld_name=args.data.tar, verbose=True) cls_error_list = np.append(cls_error_list, error.numpy()) cal_error_list = np.append(cal_error_list, ece) print( f"[final, {args.snapshot_prefix}, cls error, n = {len(cls_error_list)}] mean = {np.mean(cls_error_list*100.0):.2f}%, std = {np.std(cls_error_list*100.0):.2f}%" ) print( f"[final, {args.snapshot_prefix}, cal error, n = {len(cal_error_list)}] mean = {np.mean(cal_error_list*100.0):.2f}%, std = {np.std(cal_error_list*100.0):.2f}%" ) ## ## init student ## ## load the init student mdl_fn_init = os.path.basename( glob.glob(os.path.join(snap_root, 'model_params_*init*.index'))[0]) mdl_fn_init = mdl_fn_init[:mdl_fn_init.rfind('_')] if 'sourceonly' in mdl_fn_init: mdl_st.model_base.load_weights( os.path.join(snap_root, mdl_fn_init + '_best')) learner = LearnerClsSelf(None, None, mdl_st, None) else: assert ('advtr' in mdl_fn_init) ## init a adv model mdl_adv = getattr( model, args.train_advtr.model_advtr)(n_in=mdl_st.model_base.dim_feat) mdl_st_adv = model.DAN(mdl_st.model_base, mdl_adv) mdl_st_adv.load_weights( os.path.join(snap_root, mdl_fn_init + '_final')) ## init a learner learner = LearnerDACls(None, mdl_st_adv) ## evaluate error, ece, *_ = learner.test(ds_tar.test, ld_name=args.data.tar, verbose=True) cls_error_init_list = np.append(cls_error_init_list, error.numpy()) cal_error_init_list = np.append(cal_error_init_list, ece) print( f"[init, {args.snapshot_prefix}, cls error, n = {len(cls_error_init_list)}] mean = {np.mean(cls_error_init_list*100.0):.2f}%, std = {np.std(cls_error_init_list*100.0):.2f}%" ) print( f"[init, {args.snapshot_prefix}, cal error, n = {len(cal_error_init_list)}] mean = {np.mean(cal_error_init_list*100.0):.2f}%, std = {np.std(cal_error_init_list*100.0):.2f}%" ) ## ## teacher performance at each step ## if args.no_mid_results: continue cls_error_epoch_list, cal_error_epoch_list, prec_epoch_list, cov_epoch_list = [], [], [], [] for i_epoch in range(1, args.train.n_epochs): # ignore the last ## load print("!!!! currently load best, but may load final later") mdl_st.model_base.load_weights( os.path.join(snap_root, f'model_params_base_epoch_{i_epoch}_best')) ## cls/cal error learner = LearnerClsSelf(None, None, mdl_st, None) error, ece, *_ = learner.test(ds_tar.test, ld_name=args.data.tar, verbose=True) print(error.numpy(), ece) ## precision/coverage learner = LearnerConfPred(None, mdl_st.model_conf, mdl_st.model_base) ## set a constant mdl_st.model_conf.T = tf.Variable(1.0 - args.train_conf.eps) ## test the model prec, n_conf, n = learner.test(ds_tar.test, ld_name='tar', verbose=True) cls_error_epoch_list.append(error.numpy()) cal_error_epoch_list.append(ece) prec_epoch_list.append(prec.numpy()) cov_epoch_list.append(float(n_conf.numpy()) / float(n)) perf_epoch_list.append({ 'cls_error': np.array(cls_error_epoch_list), 'cal_error': np.array(cal_error_epoch_list), 'prec': np.array(prec_epoch_list), 'cov': np.array(cov_epoch_list) }) print() ## save fn = args.snapshot_prefix + '.pk' pickle.dump( { 'cls_error_init': cls_error_init_list, 'cal_error_init': cal_error_init_list, 'cls_error': cls_error_list, 'cal_error': cal_error_list, 'perf_epoch': perf_epoch_list }, open(fn, 'wb'))