def get_results_advtr(args, ds_src, ds_tar):
from learning import LearnerDACls
from learning import TempScalingCls as CalibratorCls
exp_name_list = glob.glob('snapshots/m2m_advtr_small_*')
error_cls_list, error_cal_list = [], []
for exp_name in exp_name_list:
params = types.SimpleNamespace()
params.save_root = exp_name
params.find_best = False
params.load_final = False
params.advtr_type = 'DANN'
params.schedule_reg_param_adv = True
print(params.save_root)
## init a base model
mdl = getattr(model, args.model.base)(num_class=args.data.n_labels,
input_shape=args.data.img_size)
## init an adv model
mdl_adv = getattr(model, 'BigAdvFNN')(n_in=mdl.dim_feat)
## init a learner
learner = LearnerDACls(params, model.DAN(mdl, mdl_adv))
## train the model
learner.train([ds_src.train, ds_dom.train], None, None)
## test the model
error_cls, error_cal, *_ = learner.test(ds_tar.test,
ld_name=args.data.tar,
verbose=True)
error_cls_list.append(error_cls.numpy())
error_cal_list.append(error_cal)
return np.array(error_cls_list), np.array(error_cal_list)
def _train_begin(self, ds_src, ds_tar, ds_dom):
"""A function called in the begning of training process. It trains an initial model for self-training.
ds_src: source dataset loader, which contains loaders for train/val/test
ds_tar: target dataset loader, which contains loaders for train/val/test
ds_dom: domain dataset loader, which contains loaders for train/val/test
"""
## initialize a base model
if self.params.init_advtr:
print("## init the teacher model with adversarial training")
model_base = self.model
model.set_trainable(model_base, True)
## init a adv model
mdl_adv = getattr(
model, self.params_init.model_advtr)(n_in=model_base.dim_feat)
## init a learner
learner = LearnerDACls(self.params_init,
model.DAN(model_base, mdl_adv),
model_name_postfix='_advtrinit')
## train the model
learner.train([ds_src.train, ds_dom.train], None, ds_tar.test)
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
print()
else:
print("## init the teacher model with sourceonly training")
model_base = self.model
model.set_trainable(model_base, True)
## init a learner
learner = LearnerCls(self.params_init,
model_base,
model_name_postfix='_sourceonlyinit')
## train the model
learner.train(ds_src.train, ds_src.val, ds_tar.test)
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
print()
print('## initialize a pseudo-labeling function')
model_base = self.model
model_conf = self.model_s.train.model_conf
## init a learner
learner = LearnerConfPred(self.params_conf,
model_conf,
model_base,
None,
model_name_postfix='_confpredinit')
## set a constant
model_conf.T = tf.Variable(1.0 - self.params_conf.eps)
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
print()
def main(args):
data_fn = 'plots/prec_cov_list.pk'
fig_fn = 'plots/prec_cov'
if os.path.exists(data_fn):
pc_data = pickle.load(open(data_fn, 'rb'))
plot_prec_cov(pc_data['T_list'], pc_data['prec_list'],
pc_data['cov_list'], fig_fn)
return
## init a snapshot path
os.makedirs(args.train.save_root, exist_ok=True)
## init logger
sys.stdout = Logger(os.path.join(args.train.save_root, 'out'))
## print args
print_args(args)
## init gpus
if not args.cpu:
print("##GPUs Available: ",
len(tf.config.experimental.list_physical_devices('GPU')))
print()
## init datasets
print("## init datasets")
ds_src = data.MultiSourceDataset(
args.data.src,
args.aug_params,
batch_size=args.data.batch_size,
val_shuffle=True,
val_aug=True,
domain_id=1,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0])
assert (len(args.aug_params) == 1) ##TODO
ds_tar = getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params[0],
val_shuffle=True,
val_aug=True,
domain_id=0,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1])
ds_dom = data.DomainDataset(
data.MultiSourceDataset(
args.data.src,
args.aug_params,
batch_size=args.data.batch_size,
val_shuffle=True,
val_aug=True,
test_aug=True, #diff
domain_id=1,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0]),
getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params[0],
val_shuffle=True,
val_aug=True,
test_aug=True, #diff
domain_id=0,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1]))
print()
####
## reliable teacher learning
####
mdl_st_base = getattr(model,
args.model.base)(num_class=args.model.n_labels,
input_shape=args.model.img_size)
#mdl_st_base = model.TempCls(mdl_st_base)
mdl_st = model.Student(args.model,
mdl_st_base,
ds_src,
ds_tar,
ideal=args.ideal)
mdl_tc_base = getattr(model,
args.model.base)(num_class=args.model.n_labels,
input_shape=args.model.img_size)
#mdl_tc_base = model.TempCls(mdl_tc_base)
mdl_tc = model.Teacher(args.model,
mdl_tc_base,
ds_src,
ds_tar,
ideal=args.ideal)
## rename
model_t = mdl_tc
model_s = mdl_st
model_c = model_s.model_base
params = args.train
params_base = args.train_base
params_advtr = args.train_advtr
params_iw = args.train_iw
params_iw_cal = args.cal_iw
params_conf = args.est_conf
i_epoch = 1
## init a model
if params.init == 'sourceonly':
##TODO: assume classification
print("## init the student model with sourceonly training")
model.set_trainable(model_c, True)
## init a learner
learner = LearnerCls(params_base,
model_c,
model_name_postfix='_sourceonlyinit')
## train the model
learner.train(ds_src.train, ds_src.val, ds_src.test)
## test the model
learner.test(ds_src.test, ld_name='src', verbose=True)
print()
elif params.init == 'advtr':
##TODO: assume classification
print("## init a base model with adversarial training")
model.set_trainable(model_c, True)
## init a adv model
mdl_adv = getattr(model,
params_advtr.model_advtr)(n_in=model_c.dim_feat)
## init a learner
learner = LearnerDACls(params_advtr,
model.DAN(model_c, mdl_adv),
model_name_postfix='_advtrinit')
## train the model
learner.train([ds_src.train, ds_dom.train], None, ds_tar.test)
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
print()
else:
raise NotImplementedError
## init iw
if model_t.train.model_conf.model_iw is not None:
print("## learn IW")
model_sd = model_t.train.model_conf.model_iw.model_sd.model
model_sd.train()
## init a learner
learner_sd = LearnerCls(params_iw,
model_sd,
model_name_postfix='_iw_epoch_%d' % (i_epoch))
## train the model
learner_sd.train(ds_dom.train, ds_dom.val, ds_dom.test)
## test the model
learner_sd.test(ds_dom.test, ld_name='domain', verbose=True)
print()
## init a calibraton model
model_sd_cal = model_t.train.model_conf.model_iw.model_sd
model_sd_cal.train()
## init a calibrator
calibrator_iw = CalibratorCls(params_iw_cal,
model_sd_cal,
model_name_postfix='_iw_cal_epoch_%d' %
(i_epoch))
## calibrate the model
calibrator_iw.train(ds_dom.val, ds_dom.val, ds_dom.test)
## test the model
calibrator_iw.test(ds_dom.test, ld_name='domain', verbose=True)
print()
## 2. learn confidence predictor
model_base = model_t.train.model_base
model_conf = model_t.train.model_conf
model_iw = model_t.train.model_conf.model_iw
model_iw_cond = model.CondIW(model_iw, model_conf, ds_src.train,
ds_tar.train)
## init a learner
learner = LearnerConfPred(params_conf,
model_conf,
model_base,
model_iw_cond,
model_name_postfix='_confpred_epoch_%d' %
(i_epoch))
## train the model
learner.train(ds_src.val, ds_src.val, ds_tar.test)
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
learner.test(ds_tar.train, ld_name='tar (train)', verbose=True)
print()
else:
model_base = model_t.train.model_base
model_conf = model_t.train.model_conf
## init a learner
learner = LearnerConfPred(params_conf,
model_conf,
model_base,
None,
model_name_postfix='_confpred_epoch_%d' %
(i_epoch))
## train the model
model_conf.T = tf.Variable(1.0 - params_conf.eps) ##TODO
print("T = %f" % (model_conf.T.numpy()))
## test the model
learner.test(ds_tar.test, ld_name='tar', verbose=True)
learner.test(ds_tar.train, ld_name='tar (train)', verbose=True)
## compute precision and coverage
T_list, prec_list, cov_list = [], [], []
for T in np.arange(0.0, 1.0, 0.01):
model_conf.T = T
prec, n_conf, n = learner.test(ds_tar.train,
ld_name='tar (train)',
verbose=True)
T_list.append(T)
prec_list.append(prec.numpy())
cov_list.append(float(n_conf) / float(n))
print(T_list)
print(prec_list)
print(cov_list)
print()
T_list = np.array(T_list)
prec_list = np.array(prec_list)
cov_list = np.array(cov_list)
pickle.dump(
{
'T_list': T_list,
'prec_list': prec_list,
'cov_list': cov_list
}, open(data_fn, 'wb'))
## init datasets
print("## init datasets")
# ds_src = data.MultiSourceDataset(args.src, [None], batch_size=args.batch_size, val_shuffle=True, val_aug=True, domain_id=1,
# color=False if args.img_size[2]==1 else True, size=args.img_size[0])
ds_tar = getattr(data, args.tar)(root=os.path.join('data', args.tar.lower()), batch_size=args.batch_size, val_shuffle=True, domain_id=0,
color=False if args.img_size[2]==1 else True, size=args.img_size[0])
## gather results
error_list, ece_list, ent_list = [], [], []
for reg in reg_param_adv_list:
## init a base model
mdl_base = getattr(model, args.model)(num_class=args.n_labels, activation='relu', input_shape=args.img_size) ##TODO: generalize
mdl_adv = getattr(model, args.model_adv)(n_in=800)#mdl_base.dim_feat)
mdl = model.DAN(mdl_base, mdl_adv)
if args.best_model:
mdl.load_weights('snapshots/%s2%s_advtr_%s%s%s_%s/model_params_best'%(src_short[0].lower(), args.tar[0].lower(), reg.replace('.', 'd'),
'_smalln' if args.smalln else '',
'_aug' if args.aug else '',
args.model))
else:
mdl.load_weights('snapshots/%s2%s_advtr_%s%s%s_%s/model_params_final'%(src_short[0].lower(), args.tar[0].lower(), reg.replace('.', 'd'),
'_smalln' if args.smalln else '',
'_aug' if args.aug else '',
args.model))
## compute error and ece
learner = LearnerDACls(None, mdl)
error, ece, ece_oc, ent = learner.test(ds_tar.test, ld_name=args.tar)
error_list.append(error.numpy())
dl_trn, vocab, lenc, ldec = dataset.load(batchSize=args.batch_size, seqLen=args.seq_len, \
path=DATA_PATH, cl=args.clip, voc=None, lenc=None, ldec=None, bertToks=args.bert_tokens)
dl_val, val_vocab, vlenc, vldec = dataset.load(batchSize =args.batch_size, seqLen = args.seq_len, \
path = VALID_PATH, cl=args.clip, voc=vocab, lenc=lenc, ldec=ldec, bertToks=args.bert_tokens)
dl_tst, tst_vocab, tlenc, tdec = dataset.load(batchSize = 1, seqLen = args.seq_len, \
path = TEST_PATH, cl=args.clip, voc=vocab, lenc=lenc, ldec=ldec, bertToks=args.bert_tokens)
device = torch.device(
"cuda" if torch.cuda.is_available() and args.cuda else "cpu")
modeldict = {
"bert": model.BertMLP(n_classes = len(lenc)),
"cnn": model.CNN(n_classes = len(lenc), vocab_size = len(vocab), emb_dim = args.embedding_dim, \
n_hidden_units = args.embedding_dim, n_filters = 100, filter_sizes = [3,4,5], device=device),
"dan": model.DAN(n_classes = len(lenc), vocab_size = len(vocab), emb_dim = args.embedding_dim, \
n_hidden_units = args.embedding_dim, device=device)
}
mdl = modeldict[args.model].to(device)
criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(mdl.parameters(), lr=args.lr)
json_obj = {"name": args.data_name}
best_loss = np.inf
for epoch in range(1, args.epochs + 1):
loss, json_loss = train(epoch, device)
json_obj["epoch {}".format(epoch)] = json_loss
if loss < best_loss:
best_loss = loss
print('* Saved')
torch.save(mdl.state_dict(), MODEL_SAV)
def main(args):
# ## init a snapshot path
# os.makedirs(args.train.save_root, exist_ok=True)
# ## init logger
# sys.stdout = Logger(os.path.join(args.train.save_root, 'out'))
# ## print args
# print_args(args)
snap_list = glob.glob(args.snapshot_prefix + '_*')
print(snap_list)
print("# experiments = ", len(snap_list))
## init gpus
if not args.cpu:
print("##GPUs Available: ",
len(tf.config.experimental.list_physical_devices('GPU')))
print()
## init datasets
print("## init datasets")
ds_src = data.MultiSourceDataset(
args.data.src,
args.aug_params,
batch_size=args.data.batch_size,
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0],
resize_pad=True if len(args.data.src) == 1
and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else
False, ##TODO: check if it's necessary
)
assert (len(args.aug_params) == 1) ##TODO
ds_tar = getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params[0],
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1])
ds_dom = data.DomainDataset(
data.MultiSourceDataset(
args.data.src,
args.aug_params,
batch_size=args.data.batch_size,
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
test_aug=True, # augment all splits
domain_id=1,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0],
resize_pad=True if len(args.data.src) == 1
and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else
False, ##TODO: check if it's necessary
),
getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params[0],
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
test_aug=True, # augment all splits
domain_id=0,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1]))
##TODO: redundant
ds_src_init = data.MultiSourceDataset(
args.data.src,
args.aug_params_init,
batch_size=args.data.batch_size,
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0], ##TODO
sample_ratio=args.data.sample_ratio[0],
resize_pad=True if len(args.data.src) == 1
and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else
False, ##TODO: check if it's necessary
)
assert (len(args.aug_params) == 1) ##TODO
ds_tar_init = getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params_init[0], ##TODO
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1])
ds_dom_init = data.DomainDataset(
data.MultiSourceDataset(
args.data.src,
args.aug_params_init,
batch_size=args.data.batch_size,
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
test_aug=True, # augment all splits
domain_id=1,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0],
resize_pad=True if len(args.data.src) == 1
and args.data.src[0] == 'MNIST' and args.data.tar == 'SVHN' else
False, ##TODO: check if it's necessary
),
getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params_init[0], ##TODO
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
test_aug=True, # augment all splits
domain_id=0,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1]))
##TODO: redundant
ds_src_self = data.MultiSourceDataset(
args.data.src,
args.aug_params,
batch_size=args.data.batch_size,
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
domain_id=1,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[0],
resize_pad=True
if len(args.data.src) == 1 and args.data.src[0] == 'MNIST'
and args.data.tar == 'SVHN' else False, ##TODO: check if...
)
assert (len(args.aug_params) == 1) ##TODO
ds_tar_self = getattr(data, args.data.tar)(
root=os.path.join('data', args.data.tar.lower()),
batch_size=args.data.batch_size,
aug_list=args.aug_params[0],
train_shuffle=True,
train_aug=True,
val_shuffle=True,
val_aug=True,
domain_id=0,
color=False if args.data.img_size[2] == 1 else True,
size=args.data.img_size[0],
sample_ratio=args.data.sample_ratio[1],
double_aug=True if args.training_type == 'selfcon' else False,
)
print()
if args.merge_train_val:
ds_src.train = data.ChainLoader(ds_src.train, ds_src.val)
ds_dom.train = data.ChainLoader(ds_dom.train, ds_dom.val)
## collect stats
cls_error_init_list, cal_error_init_list = [], []
cls_error_list, cal_error_list = [], []
perf_epoch_list = []
for snap_root in snap_list:
##
## final student
##
## a student model
mdl_st_base = getattr(model,
args.model.base)(num_class=args.model.n_labels,
input_shape=args.model.img_size)
mdl_st = model.Student(args.model,
mdl_st_base,
ds_src_self,
ds_tar_self,
ideal=args.ideal)
## load the final student
mdl_st.model_base.load_weights(
os.path.join(snap_root, 'model_params_final'))
## evaluate
learner = LearnerClsSelf(None, None, mdl_st, None)
error, ece, *_ = learner.test(ds_tar.test,
ld_name=args.data.tar,
verbose=True)
cls_error_list = np.append(cls_error_list, error.numpy())
cal_error_list = np.append(cal_error_list, ece)
print(
f"[final, {args.snapshot_prefix}, cls error, n = {len(cls_error_list)}] mean = {np.mean(cls_error_list*100.0):.2f}%, std = {np.std(cls_error_list*100.0):.2f}%"
)
print(
f"[final, {args.snapshot_prefix}, cal error, n = {len(cal_error_list)}] mean = {np.mean(cal_error_list*100.0):.2f}%, std = {np.std(cal_error_list*100.0):.2f}%"
)
##
## init student
##
## load the init student
mdl_fn_init = os.path.basename(
glob.glob(os.path.join(snap_root, 'model_params_*init*.index'))[0])
mdl_fn_init = mdl_fn_init[:mdl_fn_init.rfind('_')]
if 'sourceonly' in mdl_fn_init:
mdl_st.model_base.load_weights(
os.path.join(snap_root, mdl_fn_init + '_best'))
learner = LearnerClsSelf(None, None, mdl_st, None)
else:
assert ('advtr' in mdl_fn_init)
## init a adv model
mdl_adv = getattr(
model,
args.train_advtr.model_advtr)(n_in=mdl_st.model_base.dim_feat)
mdl_st_adv = model.DAN(mdl_st.model_base, mdl_adv)
mdl_st_adv.load_weights(
os.path.join(snap_root, mdl_fn_init + '_final'))
## init a learner
learner = LearnerDACls(None, mdl_st_adv)
## evaluate
error, ece, *_ = learner.test(ds_tar.test,
ld_name=args.data.tar,
verbose=True)
cls_error_init_list = np.append(cls_error_init_list, error.numpy())
cal_error_init_list = np.append(cal_error_init_list, ece)
print(
f"[init, {args.snapshot_prefix}, cls error, n = {len(cls_error_init_list)}] mean = {np.mean(cls_error_init_list*100.0):.2f}%, std = {np.std(cls_error_init_list*100.0):.2f}%"
)
print(
f"[init, {args.snapshot_prefix}, cal error, n = {len(cal_error_init_list)}] mean = {np.mean(cal_error_init_list*100.0):.2f}%, std = {np.std(cal_error_init_list*100.0):.2f}%"
)
##
## teacher performance at each step
##
if args.no_mid_results:
continue
cls_error_epoch_list, cal_error_epoch_list, prec_epoch_list, cov_epoch_list = [], [], [], []
for i_epoch in range(1, args.train.n_epochs): # ignore the last
## load
print("!!!! currently load best, but may load final later")
mdl_st.model_base.load_weights(
os.path.join(snap_root,
f'model_params_base_epoch_{i_epoch}_best'))
## cls/cal error
learner = LearnerClsSelf(None, None, mdl_st, None)
error, ece, *_ = learner.test(ds_tar.test,
ld_name=args.data.tar,
verbose=True)
print(error.numpy(), ece)
## precision/coverage
learner = LearnerConfPred(None, mdl_st.model_conf,
mdl_st.model_base)
## set a constant
mdl_st.model_conf.T = tf.Variable(1.0 - args.train_conf.eps)
## test the model
prec, n_conf, n = learner.test(ds_tar.test,
ld_name='tar',
verbose=True)
cls_error_epoch_list.append(error.numpy())
cal_error_epoch_list.append(ece)
prec_epoch_list.append(prec.numpy())
cov_epoch_list.append(float(n_conf.numpy()) / float(n))
perf_epoch_list.append({
'cls_error': np.array(cls_error_epoch_list),
'cal_error': np.array(cal_error_epoch_list),
'prec': np.array(prec_epoch_list),
'cov': np.array(cov_epoch_list)
})
print()
## save
fn = args.snapshot_prefix + '.pk'
pickle.dump(
{
'cls_error_init': cls_error_init_list,
'cal_error_init': cal_error_init_list,
'cls_error': cls_error_list,
'cal_error': cal_error_list,
'perf_epoch': perf_epoch_list
}, open(fn, 'wb'))