def main():
args = parse_arguments()
# argument setting
print("=== Argument Setting ===")
print("src: " + args.src)
print("tgt: " + args.tgt)
print("seed: " + str(args.seed))
print("train_seed: " + str(args.train_seed))
print("model_type: " + str(args.model))
print("max_seq_length: " + str(args.max_seq_length))
print("batch_size: " + str(args.batch_size))
print("pre_epochs: " + str(args.pre_epochs))
print("num_epochs: " + str(args.num_epochs))
print("AD weight: " + str(args.alpha))
print("KD weight: " + str(args.beta))
print("temperature: " + str(args.temperature))
set_seed(args.train_seed)
if args.model in ['roberta', 'distilroberta']:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
else:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# preprocess data
print("=== Processing datasets ===")
if args.src in ['blog', 'airline', 'imdb']:
src_x, src_y = CSV2Array(
os.path.join('data', args.src, args.src + '.csv'))
else:
src_x, src_y = XML2Array(
os.path.join('data', args.src, 'negative.review'),
os.path.join('data', args.src, 'positive.review'))
src_x, src_test_x, src_y, src_test_y = train_test_split(
src_x, src_y, test_size=0.2, stratify=src_y, random_state=args.seed)
if args.tgt in ['blog', 'airline', 'imdb']:
tgt_x, tgt_y = CSV2Array(
os.path.join('data', args.tgt, args.tgt + '.csv'))
else:
tgt_x, tgt_y = XML2Array(
os.path.join('data', args.tgt, 'negative.review'),
os.path.join('data', args.tgt, 'positive.review'))
tgt_train_x, tgt_test_y, tgt_train_y, tgt_test_y = train_test_split(
tgt_x, tgt_y, test_size=0.2, stratify=tgt_y, random_state=args.seed)
if args.model in ['roberta', 'distilroberta']:
src_features = roberta_convert_examples_to_features(
src_x, src_y, args.max_seq_length, tokenizer)
src_test_features = roberta_convert_examples_to_features(
src_test_x, src_test_y, args.max_seq_length, tokenizer)
tgt_features = roberta_convert_examples_to_features(
tgt_x, tgt_y, args.max_seq_length, tokenizer)
tgt_train_features = roberta_convert_examples_to_features(
tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer)
else:
src_features = convert_examples_to_features(src_x, src_y,
args.max_seq_length,
tokenizer)
src_test_features = convert_examples_to_features(
src_test_x, src_test_y, args.max_seq_length, tokenizer)
tgt_features = convert_examples_to_features(tgt_x, tgt_y,
args.max_seq_length,
tokenizer)
tgt_train_features = convert_examples_to_features(
tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer)
# load dataset
src_data_loader = get_data_loader(src_features, args.batch_size)
src_data_eval_loader = get_data_loader(src_test_features, args.batch_size)
tgt_data_train_loader = get_data_loader(tgt_train_features,
args.batch_size)
tgt_data_all_loader = get_data_loader(tgt_features, args.batch_size)
# load models
if args.model == 'bert':
src_encoder = BertEncoder()
tgt_encoder = BertEncoder()
src_classifier = BertClassifier()
elif args.model == 'distilbert':
src_encoder = DistilBertEncoder()
tgt_encoder = DistilBertEncoder()
src_classifier = BertClassifier()
elif args.model == 'roberta':
src_encoder = RobertaEncoder()
tgt_encoder = RobertaEncoder()
src_classifier = RobertaClassifier()
else:
src_encoder = DistilRobertaEncoder()
tgt_encoder = DistilRobertaEncoder()
src_classifier = RobertaClassifier()
discriminator = Discriminator()
if args.load:
src_encoder = init_model(args,
src_encoder,
restore=param.src_encoder_path)
src_classifier = init_model(args,
src_classifier,
restore=param.src_classifier_path)
tgt_encoder = init_model(args,
tgt_encoder,
restore=param.tgt_encoder_path)
discriminator = init_model(args,
discriminator,
restore=param.d_model_path)
else:
src_encoder = init_model(args, src_encoder)
src_classifier = init_model(args, src_classifier)
tgt_encoder = init_model(args, tgt_encoder)
discriminator = init_model(args, discriminator)
# train source model
print("=== Training classifier for source domain ===")
if args.pretrain:
src_encoder, src_classifier = pretrain(args, src_encoder,
src_classifier, src_data_loader)
# eval source model
print("=== Evaluating classifier for source domain ===")
evaluate(src_encoder, src_classifier, src_data_loader)
evaluate(src_encoder, src_classifier, src_data_eval_loader)
evaluate(src_encoder, src_classifier, tgt_data_all_loader)
for params in src_encoder.parameters():
params.requires_grad = False
for params in src_classifier.parameters():
params.requires_grad = False
# train target encoder by GAN
print("=== Training encoder for target domain ===")
if args.adapt:
tgt_encoder.load_state_dict(src_encoder.state_dict())
tgt_encoder = adapt(args, src_encoder, tgt_encoder, discriminator,
src_classifier, src_data_loader,
tgt_data_train_loader, tgt_data_all_loader)
# eval target encoder on lambda0.1 set of target dataset
print("=== Evaluating classifier for encoded target domain ===")
print(">>> source only <<<")
evaluate(src_encoder, src_classifier, tgt_data_all_loader)
print(">>> domain adaption <<<")
evaluate(tgt_encoder, src_classifier, tgt_data_all_loader)
def main(args, f):
set_seed(args.train_seed)
if args.model in ['roberta', 'distilroberta']:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
else:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# preprocess data
src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader, tgt_te = get_all_dataloader(
args, tokenizer)
# load models
if args.model == 'bert':
src_encoder = BertEncoder()
# encoder = BertEncoder()
classifier = BertClassifier()
elif args.model == 'distilbert':
src_encoder = DistilBertEncoder()
# encoder = DistilBertEncoder()
classifier = BertClassifier()
elif args.model == 'roberta':
src_encoder = RobertaEncoder()
# encoder = RobertaEncoder()
classifier = RobertaClassifier()
else:
src_encoder = DistilRobertaEncoder()
# encoder = DistilRobertaEncoder()
classifier = RobertaClassifier()
discriminator = Discriminator()
# parallel models
if torch.cuda.device_count() > 1:
print('Let\'s use {} GPUs!'.format(torch.cuda.device_count()))
src_encoder = nn.DataParallel(src_encoder)
classifier = nn.DataParallel(classifier)
# encoder = nn.DataParallel(encoder)
discriminator = nn.DataParallel(discriminator)
if args.load:
src_encoder = init_model(args,
src_encoder,
restore_path=param.src_encoder_path)
classifier = init_model(args,
classifier,
restore_path=param.src_classifier_path)
# encoder = init_model(args, encoder, restore_path=param.tgt_encoder_path)
# discriminator = init_model(args, discriminator, restore_path=param.d_model_path)
else:
src_encoder = init_model(args, src_encoder)
classifier = init_model(args, classifier)
# encoder = init_model(args, encoder)
discriminator = init_model(args, discriminator)
# train source model
if args.pretrain:
print("=== Training classifier for source domain ===")
src_encoder, classifier = pretrain(args, src_encoder, classifier,
src_loader)
# save pretrained model
# save_model(args, src_encoder, param.src_encoder_path)
# save_model(args, classifier, param.src_classifier_path)
# eval source model
print("=== Evaluating classifier for source domain ===")
evaluate(args, src_encoder, classifier, src_loader)
src_acc = evaluate(args, src_encoder, classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt}: No adapt acc on src data: {src_acc}\n')
# adapt
print("=== Adapt tgt encoder ===")
# encoder.load_state_dict(src_encoder.state_dict())
# if args.src_free:
# s_res_features = src_gmm(args, src_encoder, src_loader)
# src_loader = s_numpy_dataloader(s_res_features, args.batch_size)
# encoder = aad_adapt_src_free(args, src_encoder, encoder, discriminator,
# classifier, src_loader, tgt_train_loader, tgt_all_loader)
# else:
if args.adapt:
encoder, classifier = shot_adapt(args, src_encoder, classifier,
tgt_train_loader, tgt_all_loader,
tgt_te)
# save_model(args, encoder, param.tgt_encoder_path)
# argument setting
# print("=== Argument Setting ===")
print(
f"model_type: {args.model}; max_seq_len: {args.max_seq_length}; batch_size: {args.batch_size}; "
f"pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; src: {args.src}; tgt: {args.tgt}; "
f'src_free: {args.src_free}; dp: {args.dp}')
# eval target encoder on lambda0.1 set of target dataset
print("=== Evaluating classifier for encoded target domain ===")
print(">>> domain adaption <<<")
tgt_acc = evaluate(args, encoder, classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n')
f.write(
f"model_type: {args.model}; batch_size: {args.batch_size}; pre_epochs: {args.pre_epochs}; "
f"num_epochs: {args.num_epochs}; src_free: {args.src_free}; src: {args.src}; "
f"tgt: {args.tgt}; dp: {args.dp}\n\n")
def main():
args = parse_arguments()
# argument setting
print("=== Argument Setting ===")
print("src: " + args.src)
print("tgt: " + args.tgt)
print("alpha: " + str(args.alpha))
print("seed: " + str(args.seed))
print("train_seed: " + str(args.train_seed))
print("model_type: " + str(args.model))
print("max_seq_length: " + str(args.max_seq_length))
print("batch_size: " + str(args.batch_size))
print("num_epochs: " + str(args.num_epochs))
set_seed(args.train_seed)
if args.model == 'roberta':
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
else:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# preprocess data
print("=== Processing datasets ===")
if args.src == 'blog':
src_x, src_y = CSV2Array(os.path.join('data', args.src, 'blog.csv'))
elif args.src == 'airline':
src_x, src_y = CSV2Array(os.path.join('data', args.src, 'airline.csv'))
else:
src_x, src_y = XML2Array(
os.path.join('data', args.src, 'negative.review'),
os.path.join('data', args.src, 'positive.review'))
src_x, src_test_x, src_y, src_test_y = train_test_split(
src_x, src_y, test_size=0.2, stratify=src_y, random_state=args.seed)
if args.tgt == 'blog':
tgt_x, tgt_y = CSV2Array(os.path.join('data', args.tgt, 'blog.csv'))
elif args.tgt == 'airline':
tgt_x, tgt_y = CSV2Array(os.path.join('data', args.tgt, 'airline.csv'))
else:
tgt_x, tgt_y = XML2Array(
os.path.join('data', args.tgt, 'negative.review'),
os.path.join('data', args.tgt, 'positive.review'))
tgt_train_x, _, tgt_train_y, _ = train_test_split(tgt_x,
tgt_y,
test_size=0.2,
stratify=tgt_y,
random_state=args.seed)
if args.model == 'roberta':
src_features = roberta_convert_examples_to_features(
src_x, src_y, args.max_seq_length, tokenizer)
src_test_features = roberta_convert_examples_to_features(
src_test_x, src_test_y, args.max_seq_length, tokenizer)
tgt_features = roberta_convert_examples_to_features(
tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer)
tgt_all_features = roberta_convert_examples_to_features(
tgt_x, tgt_y, args.max_seq_length, tokenizer)
else:
src_features = convert_examples_to_features(src_x, src_y,
args.max_seq_length,
tokenizer)
src_test_features = convert_examples_to_features(
src_test_x, src_test_y, args.max_seq_length, tokenizer)
tgt_features = convert_examples_to_features(tgt_train_x, tgt_train_y,
args.max_seq_length,
tokenizer)
tgt_all_features = convert_examples_to_features(
tgt_x, tgt_y, args.max_seq_length, tokenizer)
# load dataset
src_data_loader = get_data_loader(src_features, args.batch_size)
src_data_loader_eval = get_data_loader(src_test_features, args.batch_size)
tgt_data_loader = get_data_loader(tgt_features, args.batch_size)
tgt_data_loader_all = get_data_loader(tgt_all_features, args.batch_size)
# load models
if args.model == 'bert':
encoder = BertEncoder()
cls_classifier = BertClassifier()
dom_classifier = DomainClassifier()
elif args.model == 'distilbert':
encoder = DistilBertEncoder()
cls_classifier = BertClassifier()
dom_classifier = DomainClassifier()
else:
encoder = RobertaEncoder()
cls_classifier = RobertaClassifier()
dom_classifier = RobertaDomainClassifier()
if args.load:
encoder = init_model(encoder, restore=param.encoder_path)
cls_classifier = init_model(cls_classifier,
restore=param.cls_classifier_path)
dom_classifier = init_model(dom_classifier,
restore=param.dom_classifier_path)
else:
encoder = init_model(encoder)
cls_classifier = init_model(cls_classifier)
dom_classifier = init_model(dom_classifier)
print("=== Start Training ===")
if args.train:
encoder, cls_classifier, dom_classifier = train(
args, encoder, cls_classifier, dom_classifier, src_data_loader,
src_data_loader_eval, tgt_data_loader, tgt_data_loader_all)
print("=== Evaluating classifier for encoded target domain ===")
print(">>> after training <<<")
evaluate(encoder, cls_classifier, tgt_data_loader_all)
def main(args, f):
# args = parse_arguments()
set_seed(args.train_seed)
if args.model in ['roberta', 'distilroberta']:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
else:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# preprocess data
src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader = get_all_dataloader(
args, tokenizer)
# load models
if args.model == 'bert':
encoder = BertEncoder()
src_encoder = BertEncoder()
classifier = BertClassifier()
elif args.model == 'distilbert':
encoder = DistilBertEncoder()
src_encoder = DistilBertEncoder()
classifier = BertClassifier()
elif args.model == 'roberta':
encoder = RobertaEncoder()
src_encoder = RobertaEncoder()
classifier = RobertaClassifier()
else:
encoder = DistilRobertaEncoder()
src_encoder = DistilRobertaEncoder()
classifier = RobertaClassifier()
# domain discriminator
discriminator = AdversarialNetworkCdan(param.input_dim * param.num_labels,
param.hidden_dim)
# parallel models
if torch.cuda.device_count() > 1:
print('Let\'s use {} GPUs!'.format(torch.cuda.device_count()))
encoder = nn.DataParallel(encoder)
src_encoder = nn.DataParallel(src_encoder)
classifier = nn.DataParallel(classifier)
discriminator = nn.DataParallel(discriminator)
if args.load:
encoder = init_model(args,
encoder,
restore_path=param.src_encoder_path)
src_encoder = init_model(args,
src_encoder,
restore_path=param.tgt_encoder_path)
classifier = init_model(args,
classifier,
restore_path=param.src_classifier_path)
# discriminator = init_model(args, discriminator, restore_path=param.d_model_path)
else:
encoder = init_model(args, encoder)
src_encoder = init_model(args, src_encoder)
classifier = init_model(args, classifier)
discriminator = init_model(args, discriminator)
# train source model
print("=== Pretrain encoder for source domain ===")
if args.pretrain:
encoder, classifier = pretrain(args, encoder, classifier, src_loader)
# eval source model
print("=== Evaluating classifier for source domain ===")
evaluate(args, encoder, classifier, src_loader)
src_acc = evaluate(args, encoder, classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt} no adapt acc on src data: {src_acc}\n')
# x, y = save_features(args, encoder, src_loader)
# np.savez(os.path.join(param.model_root, 's_feat_pretrain'), x, y)
# x, y = save_features(args, encoder, tgt_all_loader)
# np.savez(os.path.join(param.model_root, 't_feat_pretrain'), x, y)
# adapt
print("=== Adapt encoder for target domain ===")
src_encoder.load_state_dict(encoder.state_dict())
if args.src_free:
# use the same encoder and copy encoder to src_encoder have different baseline results
s_res_features = src_gmm(args, encoder, src_loader)
src_loader = s_numpy_dataloader(s_res_features, args.batch_size)
encoder, classifier = cdan_adapt_src_free(args, encoder, src_encoder,
discriminator, classifier,
src_loader, tgt_train_loader,
tgt_all_loader)
elif args.data_free:
s_res_features = src_gmm(args, encoder, src_loader)
t_res_features = tgt_gmm(encoder, tgt_all_loader, 1)
src_loader = s_numpy_dataloader(s_res_features, args.batch_size)
tgt_train_loader = t_numpy_dataloader(t_res_features, args.batch_size)
encoder, classifier = cdan_adapt_data_free(args, encoder,
discriminator, classifier,
src_loader,
tgt_train_loader,
tgt_all_loader)
else:
encoder, classifier = cdan_adapt(args, encoder, discriminator,
classifier, src_loader,
tgt_train_loader, tgt_all_loader)
# x, y = save_features(args, encoder, src_loader)
# np.savez(os.path.join(param.model_root, 's_feat_adapt_cdan'), x, y)
# x, y = save_features(args, encoder, tgt_all_loader)
# np.savez(os.path.join(param.model_root, 't_feat_adapt_cdan'), x, y)
# argument setting
print(
f"model_type: {args.model}; batch_size: {args.batch_size}; data_free: {args.data_free}; "
f"src_free: {args.src_free}; pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; "
f"src: {args.src}; tgt: {args.tgt}; kd: {args.kd}; dp: {args.dp}; ent: {args.ent}"
)
# eval target encoder on lambda0.1 set of target dataset
print("=== Evaluating classifier for encoded target domain ===")
print(">>> domain adaption <<<")
tgt_acc = evaluate(args, encoder, classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n')
f.write(
f"model_type: {args.model}; batch_size: {args.batch_size}; data_free: {args.data_free}; "
f"src_free: {args.src_free}; pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; "
f"src: {args.src}; tgt: {args.tgt}; kd: {args.kd}; dp: {args.dp}; ent: {args.ent}\n\n"
)
def main(args, f):
# args = parse_arguments()
set_seed(args.train_seed)
if args.model in ['roberta', 'distilroberta']:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
else:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# preprocess data
src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader = get_all_dataloader(
args, tokenizer)
# load models
if args.model == 'bert':
src_encoder = BertEncoder()
tgt_encoder = BertEncoder()
src_classifier = BertClassifier()
elif args.model == 'distilbert':
src_encoder = DistilBertEncoder()
tgt_encoder = DistilBertEncoder()
src_classifier = BertClassifier()
elif args.model == 'roberta':
src_encoder = RobertaEncoder()
tgt_encoder = RobertaEncoder()
src_classifier = RobertaClassifier()
else:
src_encoder = DistilRobertaEncoder()
tgt_encoder = DistilRobertaEncoder()
src_classifier = RobertaClassifier()
discriminator = Discriminator() # output dims is 2 instead of 1
if args.load:
src_encoder = init_model(args,
src_encoder,
restore_path=param.src_encoder_path)
src_classifier = init_model(args,
src_classifier,
restore_path=param.src_classifier_path)
tgt_encoder = init_model(args,
tgt_encoder,
restore_path=param.tgt_encoder_path)
discriminator = init_model(args,
discriminator,
restore_path=param.d_model_path)
else:
src_encoder = init_model(args, src_encoder)
src_classifier = init_model(args, src_classifier)
tgt_encoder = init_model(args, tgt_encoder)
discriminator = init_model(args, discriminator)
# parallel models
if torch.cuda.device_count() > 1:
print('Let\'s use {} GPUs!'.format(torch.cuda.device_count()))
src_encoder = nn.DataParallel(src_encoder)
src_classifier = nn.DataParallel(src_classifier)
tgt_encoder = nn.DataParallel(tgt_encoder)
discriminator = nn.DataParallel(discriminator)
# train source model
print("=== Training classifier for source domain ===")
if args.pretrain:
src_encoder, src_classifier = pretrain(args, src_encoder,
src_classifier, src_loader)
# eval source model
print("=== Evaluating classifier for source domain ===")
evaluate(args, src_encoder, src_classifier, src_loader)
src_acc = evaluate(args, src_encoder, src_classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt}: No adapt acc on src data: {src_acc}\n')
for params in src_encoder.parameters():
params.requires_grad = False
# train target encoder by ADDA
print("=== Training encoder for target domain ===")
if args.adapt:
tgt_encoder.load_state_dict(src_encoder.state_dict())
tgt_encoder = adda_adapt(args, src_encoder, tgt_encoder, discriminator,
src_loader, tgt_train_loader)
# argument setting
print(
f"model_type: {args.model}; max_seq_len: {args.max_seq_length}; batch_size: {args.batch_size}; "
f"pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; src: {args.src}; tgt: {args.tgt}"
)
# eval target encoder on lambda0.1 set of target dataset
print("=== Evaluating classifier for encoded target domain ===")
print(">>> domain adaption <<<")
tgt_acc = evaluate(args, tgt_encoder, src_classifier, tgt_all_loader)
f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n')
f.write(
f"model_type: {args.model}; batch_size: {args.batch_size}; pre_epochs: {args.pre_epochs}; "
f"num_epochs: {args.num_epochs}; src_free: {args.src_free}; src: {args.src}; "
f"tgt: {args.tgt};\n\n")