def main(): args = parse_arguments() # argument setting print("=== Argument Setting ===") print("src: " + args.src) print("tgt: " + args.tgt) print("seed: " + str(args.seed)) print("train_seed: " + str(args.train_seed)) print("model_type: " + str(args.model)) print("max_seq_length: " + str(args.max_seq_length)) print("batch_size: " + str(args.batch_size)) print("pre_epochs: " + str(args.pre_epochs)) print("num_epochs: " + str(args.num_epochs)) print("AD weight: " + str(args.alpha)) print("KD weight: " + str(args.beta)) print("temperature: " + str(args.temperature)) set_seed(args.train_seed) if args.model in ['roberta', 'distilroberta']: tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # preprocess data print("=== Processing datasets ===") if args.src in ['blog', 'airline', 'imdb']: src_x, src_y = CSV2Array( os.path.join('data', args.src, args.src + '.csv')) else: src_x, src_y = XML2Array( os.path.join('data', args.src, 'negative.review'), os.path.join('data', args.src, 'positive.review')) src_x, src_test_x, src_y, src_test_y = train_test_split( src_x, src_y, test_size=0.2, stratify=src_y, random_state=args.seed) if args.tgt in ['blog', 'airline', 'imdb']: tgt_x, tgt_y = CSV2Array( os.path.join('data', args.tgt, args.tgt + '.csv')) else: tgt_x, tgt_y = XML2Array( os.path.join('data', args.tgt, 'negative.review'), os.path.join('data', args.tgt, 'positive.review')) tgt_train_x, tgt_test_y, tgt_train_y, tgt_test_y = train_test_split( tgt_x, tgt_y, test_size=0.2, stratify=tgt_y, random_state=args.seed) if args.model in ['roberta', 'distilroberta']: src_features = roberta_convert_examples_to_features( src_x, src_y, args.max_seq_length, tokenizer) src_test_features = roberta_convert_examples_to_features( src_test_x, src_test_y, args.max_seq_length, tokenizer) tgt_features = roberta_convert_examples_to_features( tgt_x, tgt_y, args.max_seq_length, tokenizer) tgt_train_features = roberta_convert_examples_to_features( tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer) else: src_features = convert_examples_to_features(src_x, src_y, args.max_seq_length, tokenizer) src_test_features = convert_examples_to_features( src_test_x, src_test_y, args.max_seq_length, tokenizer) tgt_features = convert_examples_to_features(tgt_x, tgt_y, args.max_seq_length, tokenizer) tgt_train_features = convert_examples_to_features( tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer) # load dataset src_data_loader = get_data_loader(src_features, args.batch_size) src_data_eval_loader = get_data_loader(src_test_features, args.batch_size) tgt_data_train_loader = get_data_loader(tgt_train_features, args.batch_size) tgt_data_all_loader = get_data_loader(tgt_features, args.batch_size) # load models if args.model == 'bert': src_encoder = BertEncoder() tgt_encoder = BertEncoder() src_classifier = BertClassifier() elif args.model == 'distilbert': src_encoder = DistilBertEncoder() tgt_encoder = DistilBertEncoder() src_classifier = BertClassifier() elif args.model == 'roberta': src_encoder = RobertaEncoder() tgt_encoder = RobertaEncoder() src_classifier = RobertaClassifier() else: src_encoder = DistilRobertaEncoder() tgt_encoder = DistilRobertaEncoder() src_classifier = RobertaClassifier() discriminator = Discriminator() if args.load: src_encoder = init_model(args, src_encoder, restore=param.src_encoder_path) src_classifier = init_model(args, src_classifier, restore=param.src_classifier_path) tgt_encoder = init_model(args, tgt_encoder, restore=param.tgt_encoder_path) discriminator = init_model(args, discriminator, restore=param.d_model_path) else: src_encoder = init_model(args, src_encoder) src_classifier = init_model(args, src_classifier) tgt_encoder = init_model(args, tgt_encoder) discriminator = init_model(args, discriminator) # train source model print("=== Training classifier for source domain ===") if args.pretrain: src_encoder, src_classifier = pretrain(args, src_encoder, src_classifier, src_data_loader) # eval source model print("=== Evaluating classifier for source domain ===") evaluate(src_encoder, src_classifier, src_data_loader) evaluate(src_encoder, src_classifier, src_data_eval_loader) evaluate(src_encoder, src_classifier, tgt_data_all_loader) for params in src_encoder.parameters(): params.requires_grad = False for params in src_classifier.parameters(): params.requires_grad = False # train target encoder by GAN print("=== Training encoder for target domain ===") if args.adapt: tgt_encoder.load_state_dict(src_encoder.state_dict()) tgt_encoder = adapt(args, src_encoder, tgt_encoder, discriminator, src_classifier, src_data_loader, tgt_data_train_loader, tgt_data_all_loader) # eval target encoder on lambda0.1 set of target dataset print("=== Evaluating classifier for encoded target domain ===") print(">>> source only <<<") evaluate(src_encoder, src_classifier, tgt_data_all_loader) print(">>> domain adaption <<<") evaluate(tgt_encoder, src_classifier, tgt_data_all_loader)
def main(args, f): set_seed(args.train_seed) if args.model in ['roberta', 'distilroberta']: tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # preprocess data src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader, tgt_te = get_all_dataloader( args, tokenizer) # load models if args.model == 'bert': src_encoder = BertEncoder() # encoder = BertEncoder() classifier = BertClassifier() elif args.model == 'distilbert': src_encoder = DistilBertEncoder() # encoder = DistilBertEncoder() classifier = BertClassifier() elif args.model == 'roberta': src_encoder = RobertaEncoder() # encoder = RobertaEncoder() classifier = RobertaClassifier() else: src_encoder = DistilRobertaEncoder() # encoder = DistilRobertaEncoder() classifier = RobertaClassifier() discriminator = Discriminator() # parallel models if torch.cuda.device_count() > 1: print('Let\'s use {} GPUs!'.format(torch.cuda.device_count())) src_encoder = nn.DataParallel(src_encoder) classifier = nn.DataParallel(classifier) # encoder = nn.DataParallel(encoder) discriminator = nn.DataParallel(discriminator) if args.load: src_encoder = init_model(args, src_encoder, restore_path=param.src_encoder_path) classifier = init_model(args, classifier, restore_path=param.src_classifier_path) # encoder = init_model(args, encoder, restore_path=param.tgt_encoder_path) # discriminator = init_model(args, discriminator, restore_path=param.d_model_path) else: src_encoder = init_model(args, src_encoder) classifier = init_model(args, classifier) # encoder = init_model(args, encoder) discriminator = init_model(args, discriminator) # train source model if args.pretrain: print("=== Training classifier for source domain ===") src_encoder, classifier = pretrain(args, src_encoder, classifier, src_loader) # save pretrained model # save_model(args, src_encoder, param.src_encoder_path) # save_model(args, classifier, param.src_classifier_path) # eval source model print("=== Evaluating classifier for source domain ===") evaluate(args, src_encoder, classifier, src_loader) src_acc = evaluate(args, src_encoder, classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt}: No adapt acc on src data: {src_acc}\n') # adapt print("=== Adapt tgt encoder ===") # encoder.load_state_dict(src_encoder.state_dict()) # if args.src_free: # s_res_features = src_gmm(args, src_encoder, src_loader) # src_loader = s_numpy_dataloader(s_res_features, args.batch_size) # encoder = aad_adapt_src_free(args, src_encoder, encoder, discriminator, # classifier, src_loader, tgt_train_loader, tgt_all_loader) # else: if args.adapt: encoder, classifier = shot_adapt(args, src_encoder, classifier, tgt_train_loader, tgt_all_loader, tgt_te) # save_model(args, encoder, param.tgt_encoder_path) # argument setting # print("=== Argument Setting ===") print( f"model_type: {args.model}; max_seq_len: {args.max_seq_length}; batch_size: {args.batch_size}; " f"pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; src: {args.src}; tgt: {args.tgt}; " f'src_free: {args.src_free}; dp: {args.dp}') # eval target encoder on lambda0.1 set of target dataset print("=== Evaluating classifier for encoded target domain ===") print(">>> domain adaption <<<") tgt_acc = evaluate(args, encoder, classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n') f.write( f"model_type: {args.model}; batch_size: {args.batch_size}; pre_epochs: {args.pre_epochs}; " f"num_epochs: {args.num_epochs}; src_free: {args.src_free}; src: {args.src}; " f"tgt: {args.tgt}; dp: {args.dp}\n\n")
def main(): args = parse_arguments() # argument setting print("=== Argument Setting ===") print("src: " + args.src) print("tgt: " + args.tgt) print("alpha: " + str(args.alpha)) print("seed: " + str(args.seed)) print("train_seed: " + str(args.train_seed)) print("model_type: " + str(args.model)) print("max_seq_length: " + str(args.max_seq_length)) print("batch_size: " + str(args.batch_size)) print("num_epochs: " + str(args.num_epochs)) set_seed(args.train_seed) if args.model == 'roberta': tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # preprocess data print("=== Processing datasets ===") if args.src == 'blog': src_x, src_y = CSV2Array(os.path.join('data', args.src, 'blog.csv')) elif args.src == 'airline': src_x, src_y = CSV2Array(os.path.join('data', args.src, 'airline.csv')) else: src_x, src_y = XML2Array( os.path.join('data', args.src, 'negative.review'), os.path.join('data', args.src, 'positive.review')) src_x, src_test_x, src_y, src_test_y = train_test_split( src_x, src_y, test_size=0.2, stratify=src_y, random_state=args.seed) if args.tgt == 'blog': tgt_x, tgt_y = CSV2Array(os.path.join('data', args.tgt, 'blog.csv')) elif args.tgt == 'airline': tgt_x, tgt_y = CSV2Array(os.path.join('data', args.tgt, 'airline.csv')) else: tgt_x, tgt_y = XML2Array( os.path.join('data', args.tgt, 'negative.review'), os.path.join('data', args.tgt, 'positive.review')) tgt_train_x, _, tgt_train_y, _ = train_test_split(tgt_x, tgt_y, test_size=0.2, stratify=tgt_y, random_state=args.seed) if args.model == 'roberta': src_features = roberta_convert_examples_to_features( src_x, src_y, args.max_seq_length, tokenizer) src_test_features = roberta_convert_examples_to_features( src_test_x, src_test_y, args.max_seq_length, tokenizer) tgt_features = roberta_convert_examples_to_features( tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer) tgt_all_features = roberta_convert_examples_to_features( tgt_x, tgt_y, args.max_seq_length, tokenizer) else: src_features = convert_examples_to_features(src_x, src_y, args.max_seq_length, tokenizer) src_test_features = convert_examples_to_features( src_test_x, src_test_y, args.max_seq_length, tokenizer) tgt_features = convert_examples_to_features(tgt_train_x, tgt_train_y, args.max_seq_length, tokenizer) tgt_all_features = convert_examples_to_features( tgt_x, tgt_y, args.max_seq_length, tokenizer) # load dataset src_data_loader = get_data_loader(src_features, args.batch_size) src_data_loader_eval = get_data_loader(src_test_features, args.batch_size) tgt_data_loader = get_data_loader(tgt_features, args.batch_size) tgt_data_loader_all = get_data_loader(tgt_all_features, args.batch_size) # load models if args.model == 'bert': encoder = BertEncoder() cls_classifier = BertClassifier() dom_classifier = DomainClassifier() elif args.model == 'distilbert': encoder = DistilBertEncoder() cls_classifier = BertClassifier() dom_classifier = DomainClassifier() else: encoder = RobertaEncoder() cls_classifier = RobertaClassifier() dom_classifier = RobertaDomainClassifier() if args.load: encoder = init_model(encoder, restore=param.encoder_path) cls_classifier = init_model(cls_classifier, restore=param.cls_classifier_path) dom_classifier = init_model(dom_classifier, restore=param.dom_classifier_path) else: encoder = init_model(encoder) cls_classifier = init_model(cls_classifier) dom_classifier = init_model(dom_classifier) print("=== Start Training ===") if args.train: encoder, cls_classifier, dom_classifier = train( args, encoder, cls_classifier, dom_classifier, src_data_loader, src_data_loader_eval, tgt_data_loader, tgt_data_loader_all) print("=== Evaluating classifier for encoded target domain ===") print(">>> after training <<<") evaluate(encoder, cls_classifier, tgt_data_loader_all)
def main(args, f): # args = parse_arguments() set_seed(args.train_seed) if args.model in ['roberta', 'distilroberta']: tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # preprocess data src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader = get_all_dataloader( args, tokenizer) # load models if args.model == 'bert': encoder = BertEncoder() src_encoder = BertEncoder() classifier = BertClassifier() elif args.model == 'distilbert': encoder = DistilBertEncoder() src_encoder = DistilBertEncoder() classifier = BertClassifier() elif args.model == 'roberta': encoder = RobertaEncoder() src_encoder = RobertaEncoder() classifier = RobertaClassifier() else: encoder = DistilRobertaEncoder() src_encoder = DistilRobertaEncoder() classifier = RobertaClassifier() # domain discriminator discriminator = AdversarialNetworkCdan(param.input_dim * param.num_labels, param.hidden_dim) # parallel models if torch.cuda.device_count() > 1: print('Let\'s use {} GPUs!'.format(torch.cuda.device_count())) encoder = nn.DataParallel(encoder) src_encoder = nn.DataParallel(src_encoder) classifier = nn.DataParallel(classifier) discriminator = nn.DataParallel(discriminator) if args.load: encoder = init_model(args, encoder, restore_path=param.src_encoder_path) src_encoder = init_model(args, src_encoder, restore_path=param.tgt_encoder_path) classifier = init_model(args, classifier, restore_path=param.src_classifier_path) # discriminator = init_model(args, discriminator, restore_path=param.d_model_path) else: encoder = init_model(args, encoder) src_encoder = init_model(args, src_encoder) classifier = init_model(args, classifier) discriminator = init_model(args, discriminator) # train source model print("=== Pretrain encoder for source domain ===") if args.pretrain: encoder, classifier = pretrain(args, encoder, classifier, src_loader) # eval source model print("=== Evaluating classifier for source domain ===") evaluate(args, encoder, classifier, src_loader) src_acc = evaluate(args, encoder, classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt} no adapt acc on src data: {src_acc}\n') # x, y = save_features(args, encoder, src_loader) # np.savez(os.path.join(param.model_root, 's_feat_pretrain'), x, y) # x, y = save_features(args, encoder, tgt_all_loader) # np.savez(os.path.join(param.model_root, 't_feat_pretrain'), x, y) # adapt print("=== Adapt encoder for target domain ===") src_encoder.load_state_dict(encoder.state_dict()) if args.src_free: # use the same encoder and copy encoder to src_encoder have different baseline results s_res_features = src_gmm(args, encoder, src_loader) src_loader = s_numpy_dataloader(s_res_features, args.batch_size) encoder, classifier = cdan_adapt_src_free(args, encoder, src_encoder, discriminator, classifier, src_loader, tgt_train_loader, tgt_all_loader) elif args.data_free: s_res_features = src_gmm(args, encoder, src_loader) t_res_features = tgt_gmm(encoder, tgt_all_loader, 1) src_loader = s_numpy_dataloader(s_res_features, args.batch_size) tgt_train_loader = t_numpy_dataloader(t_res_features, args.batch_size) encoder, classifier = cdan_adapt_data_free(args, encoder, discriminator, classifier, src_loader, tgt_train_loader, tgt_all_loader) else: encoder, classifier = cdan_adapt(args, encoder, discriminator, classifier, src_loader, tgt_train_loader, tgt_all_loader) # x, y = save_features(args, encoder, src_loader) # np.savez(os.path.join(param.model_root, 's_feat_adapt_cdan'), x, y) # x, y = save_features(args, encoder, tgt_all_loader) # np.savez(os.path.join(param.model_root, 't_feat_adapt_cdan'), x, y) # argument setting print( f"model_type: {args.model}; batch_size: {args.batch_size}; data_free: {args.data_free}; " f"src_free: {args.src_free}; pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; " f"src: {args.src}; tgt: {args.tgt}; kd: {args.kd}; dp: {args.dp}; ent: {args.ent}" ) # eval target encoder on lambda0.1 set of target dataset print("=== Evaluating classifier for encoded target domain ===") print(">>> domain adaption <<<") tgt_acc = evaluate(args, encoder, classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n') f.write( f"model_type: {args.model}; batch_size: {args.batch_size}; data_free: {args.data_free}; " f"src_free: {args.src_free}; pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; " f"src: {args.src}; tgt: {args.tgt}; kd: {args.kd}; dp: {args.dp}; ent: {args.ent}\n\n" )
def main(args, f): # args = parse_arguments() set_seed(args.train_seed) if args.model in ['roberta', 'distilroberta']: tokenizer = RobertaTokenizer.from_pretrained('roberta-base') else: tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # preprocess data src_eval_loader, src_loader, tgt_all_loader, tgt_train_loader = get_all_dataloader( args, tokenizer) # load models if args.model == 'bert': src_encoder = BertEncoder() tgt_encoder = BertEncoder() src_classifier = BertClassifier() elif args.model == 'distilbert': src_encoder = DistilBertEncoder() tgt_encoder = DistilBertEncoder() src_classifier = BertClassifier() elif args.model == 'roberta': src_encoder = RobertaEncoder() tgt_encoder = RobertaEncoder() src_classifier = RobertaClassifier() else: src_encoder = DistilRobertaEncoder() tgt_encoder = DistilRobertaEncoder() src_classifier = RobertaClassifier() discriminator = Discriminator() # output dims is 2 instead of 1 if args.load: src_encoder = init_model(args, src_encoder, restore_path=param.src_encoder_path) src_classifier = init_model(args, src_classifier, restore_path=param.src_classifier_path) tgt_encoder = init_model(args, tgt_encoder, restore_path=param.tgt_encoder_path) discriminator = init_model(args, discriminator, restore_path=param.d_model_path) else: src_encoder = init_model(args, src_encoder) src_classifier = init_model(args, src_classifier) tgt_encoder = init_model(args, tgt_encoder) discriminator = init_model(args, discriminator) # parallel models if torch.cuda.device_count() > 1: print('Let\'s use {} GPUs!'.format(torch.cuda.device_count())) src_encoder = nn.DataParallel(src_encoder) src_classifier = nn.DataParallel(src_classifier) tgt_encoder = nn.DataParallel(tgt_encoder) discriminator = nn.DataParallel(discriminator) # train source model print("=== Training classifier for source domain ===") if args.pretrain: src_encoder, src_classifier = pretrain(args, src_encoder, src_classifier, src_loader) # eval source model print("=== Evaluating classifier for source domain ===") evaluate(args, src_encoder, src_classifier, src_loader) src_acc = evaluate(args, src_encoder, src_classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt}: No adapt acc on src data: {src_acc}\n') for params in src_encoder.parameters(): params.requires_grad = False # train target encoder by ADDA print("=== Training encoder for target domain ===") if args.adapt: tgt_encoder.load_state_dict(src_encoder.state_dict()) tgt_encoder = adda_adapt(args, src_encoder, tgt_encoder, discriminator, src_loader, tgt_train_loader) # argument setting print( f"model_type: {args.model}; max_seq_len: {args.max_seq_length}; batch_size: {args.batch_size}; " f"pre_epochs: {args.pre_epochs}; num_epochs: {args.num_epochs}; src: {args.src}; tgt: {args.tgt}" ) # eval target encoder on lambda0.1 set of target dataset print("=== Evaluating classifier for encoded target domain ===") print(">>> domain adaption <<<") tgt_acc = evaluate(args, tgt_encoder, src_classifier, tgt_all_loader) f.write(f'{args.src} -> {args.tgt}: DA acc on tgt data: {tgt_acc}\n') f.write( f"model_type: {args.model}; batch_size: {args.batch_size}; pre_epochs: {args.pre_epochs}; " f"num_epochs: {args.num_epochs}; src_free: {args.src_free}; src: {args.src}; " f"tgt: {args.tgt};\n\n")