def evaluate(data_loader, model, label_to_ix): ix_to_label = {v: k for k, v in label_to_ix.items()} correct = 0 total = 0 model.eval() loader_it = iter(data_loader) num_it = len(data_loader) instances = [] for j in range(num_it): mention_inputs, features, sentences, char_inputs, targets = utils.endless_get_next_batch( data_loader, loader_it) targets = utils.get_var(targets) pred = model(mention_inputs, char_inputs) _, y_pred = torch.max(pred, 1) total += targets.size(0) # correct += (y_pred == targets).sum().sample_data[0] correct += (y_pred == targets).sum().item() # output evaluate pred_numpy = (y_pred.data).cpu().numpy() y_pred_labels = [ix_to_label[ix] for ix in pred_numpy] assert len(y_pred_labels) == len( features), 'y_pred_labels and features have different lengths' for i, pred_label in enumerate(y_pred_labels): features[i][5] = pred_label instances.append(features[i]) acc = 100.0 * correct / total return acc, instances
def get_visual_features(num_iter, test_loaders, test_iters, F_d): visual_features, senti_labels = None, None with torch.no_grad(): for _ in tqdm(range(num_iter)): for domain in opt.domains: d_inputs, targets = utils.endless_get_next_batch( test_loaders, test_iters, domain) private_features = F_d[domain](d_inputs) if visual_features is None: visual_features = private_features senti_labels = targets else: visual_features = torch.cat( [visual_features, private_features], 0) senti_labels = torch.cat([senti_labels, targets], 0) return visual_features, senti_labels
dict_num_iter = len(dict_loader) #start training dictionary logging.info("batch_size: %s, dict_num_iter %s, train num_iter %s" % (str(opt.batch_size), str(dict_num_iter), str(num_iter))) for epoch in range(opt.dict_iteration): epoch_start = time.time() # sum_dict_cost = 0.0 correct_1, total_1 = 0, 0 model.train() for i in range(dict_num_iter): dict_inputs, _, dict_sentences, dict_char_inputs, dict_targets = utils.endless_get_next_batch( dict_loader, dict_iter) dict_targets = utils.get_var(dict_targets) dict_batch_output = model(dict_inputs, dict_char_inputs) dict_cost = criterion(dict_batch_output, dict_targets) # sum_dict_cost += dict_cost.item() # for dict training accuracy total_1 += len(dict_inputs[1]) _, dict_pred = torch.max(dict_batch_output, 1) correct_1 += (dict_pred == dict_targets).sum().item() dict_cost.backward() optimizer.step() model.zero_grad()
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets): """ train_sets, dev_sets, test_sets: dict[domain] -> AmazonDataset For unlabeled domains, no train_sets are available """ # dataset loaders train_loaders, unlabeled_loaders = {}, {} train_iters, unlabeled_iters = {}, {} dev_loaders, test_loaders = {}, {} my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate for domain in opt.domains: train_loaders[domain] = DataLoader(train_sets[domain], opt.batch_size, shuffle=True, collate_fn=my_collate) train_iters[domain] = iter(train_loaders[domain]) for domain in opt.dev_domains: dev_loaders[domain] = DataLoader(dev_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) test_loaders[domain] = DataLoader(test_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) for domain in opt.all_domains: if domain in opt.unlabeled_domains: uset = unlabeled_sets[domain] else: # for labeled domains, consider which data to use as unlabeled set if opt.unlabeled_data == 'both': uset = ConcatDataset([train_sets[domain], unlabeled_sets[domain]]) elif opt.unlabeled_data == 'unlabeled': uset = unlabeled_sets[domain] elif opt.unlabeled_data == 'train': uset = train_sets[domain] else: raise Exception('Unknown options for the unlabeled data usage: {}'.format(opt.unlabeled_data)) unlabeled_loaders[domain] = DataLoader(uset, opt.batch_size, shuffle=True, collate_fn=my_collate) unlabeled_iters[domain] = iter(unlabeled_loaders[domain]) # models F_s = None C, D = None, None if opt.model.lower() == 'dan': F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) elif opt.model.lower() == 'lstm': F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.dropout, opt.bdrnn, opt.attn) elif opt.model.lower() == 'cnn': F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) else: raise Exception('Unknown model architecture {}'.format(opt.model)) C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn) F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain( *map(list, [F_s.parameters() if F_s else [], C.parameters()] + [])), lr=opt.learning_rate) optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate) # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() D.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # D iterations utils.freeze_net(F_s) utils.freeze_net(C) utils.unfreeze_net(D) # WGAN n_critic trick since D trains slower n_critic = opt.n_critic if opt.wgan_trick: if opt.n_critic > 0 and ((epoch == 0 and i < 25) or i % 500 == 0): n_critic = 100 for _ in range(n_critic): D.zero_grad() loss_d = {} # train on both labeled and unlabeled domains for domain in opt.all_domains: # targets not used d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1])) shared_feat = F_s(d_inputs) d_outputs = D(shared_feat) # D accuracy _, pred = torch.max(d_outputs, 1) d_total += len(d_inputs[1]) if opt.loss.lower() == 'l2': _, tgt_indices = torch.max(d_targets, 1) d_correct += (pred == tgt_indices).sum().item() l_d = functional.mse_loss(d_outputs, d_targets) l_d.backward() else: d_correct += (pred == d_targets).sum().item() l_d = functional.nll_loss(d_outputs, d_targets) l_d.backward() loss_d[domain] = l_d.item() optimizerD.step() # F&C iteration utils.unfreeze_net(F_s) utils.unfreeze_net(C) utils.freeze_net(D) if opt.fix_emb: utils.freeze_net(F_s.word_emb) F_s.zero_grad() C.zero_grad() for domain in opt.domains: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, domain) targets = targets.to(opt.device) shared_feat = F_s(inputs) domain_feat = torch.zeros(len(targets), opt.domain_hidden_size).to(opt.device) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() # update F with D gradients on all domains for domain in opt.all_domains: d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) shared_feat = F_s(d_inputs) d_outputs = D(shared_feat) if opt.loss.lower() == 'gr': d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1])) l_d = functional.nll_loss(d_outputs, d_targets) if opt.lambd > 0: l_d *= -opt.lambd elif opt.loss.lower() == 'bs': d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs[1])) l_d = functional.kl_div(d_outputs, d_targets, size_average=False) if opt.lambd > 0: l_d *= opt.lambd elif opt.loss.lower() == 'l2': d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs[1])) l_d = functional.mse_loss(d_outputs, d_targets) if opt.lambd > 0: l_d *= opt.lambd l_d.backward() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) if avg_acc > best_avg_acc: log.info('New best average validation accuracy: {}'.format(avg_acc)) best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) torch.save(F_s.state_dict(), '{}/netF_s.pth'.format(opt.model_save_file)) torch.save(C.state_dict(), '{}/netC.pth'.format(opt.model_save_file)) torch.save(D.state_dict(), '{}/netD.pth'.format(opt.model_save_file)) # end of training log.info('Best average validation accuracy: {}'.format(best_avg_acc)) return best_acc
def train(vocabs, char_vocab, tag_vocab, train_sets, dev_sets, test_sets, unlabeled_sets): """ train_sets, dev_sets, test_sets: dict[lang] -> AmazonDataset For unlabeled langs, no train_sets are available """ # dataset loaders train_loaders, unlabeled_loaders = {}, {} train_iters, unlabeled_iters, d_unlabeled_iters = {}, {}, {} dev_loaders, test_loaders = {}, {} my_collate = utils.sorted_collate if opt.model=='lstm' else utils.unsorted_collate for lang in opt.langs: train_loaders[lang] = DataLoader(train_sets[lang], opt.batch_size, shuffle=True, collate_fn = my_collate) train_iters[lang] = iter(train_loaders[lang]) for lang in opt.dev_langs: dev_loaders[lang] = DataLoader(dev_sets[lang], opt.batch_size, shuffle=False, collate_fn = my_collate) test_loaders[lang] = DataLoader(test_sets[lang], opt.batch_size, shuffle=False, collate_fn = my_collate) for lang in opt.all_langs: if lang in opt.unlabeled_langs: uset = unlabeled_sets[lang] else: # for labeled langs, consider which data to use as unlabeled set if opt.unlabeled_data == 'both': uset = ConcatDataset([train_sets[lang], unlabeled_sets[lang]]) elif opt.unlabeled_data == 'unlabeled': uset = unlabeled_sets[lang] elif opt.unlabeled_data == 'train': uset = train_sets[lang] else: raise Exception(f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}') unlabeled_loaders[lang] = DataLoader(uset, opt.batch_size, shuffle=True, collate_fn = my_collate) unlabeled_iters[lang] = iter(unlabeled_loaders[lang]) d_unlabeled_iters[lang] = iter(unlabeled_loaders[lang]) # embeddings emb = MultiLangWordEmb(vocabs, char_vocab, opt.use_wordemb, opt.use_charemb).to(opt.device) # models F_s = None F_p = None C, D = None, None num_experts = len(opt.langs)+1 if opt.expert_sp else len(opt.langs) if opt.model.lower() == 'lstm': if opt.shared_hidden_size > 0: F_s = LSTMFeatureExtractor(opt.total_emb_size, opt.F_layers, opt.shared_hidden_size, opt.word_dropout, opt.dropout, opt.bdrnn) if opt.private_hidden_size > 0: if not opt.concat_sp: assert opt.shared_hidden_size == opt.private_hidden_size, "shared dim != private dim when using add_sp!" F_p = nn.Sequential( LSTMFeatureExtractor(opt.total_emb_size, opt.F_layers, opt.private_hidden_size, opt.word_dropout, opt.dropout, opt.bdrnn), MixtureOfExperts(opt.MoE_layers, opt.private_hidden_size, len(opt.langs), opt.private_hidden_size, opt.private_hidden_size, opt.dropout, opt.MoE_bn, False) ) else: raise Exception(f'Unknown model architecture {opt.model}') if opt.C_MoE: C = SpMixtureOfExperts(opt.C_layers, opt.shared_hidden_size, opt.private_hidden_size, opt.concat_sp, num_experts, opt.shared_hidden_size + opt.private_hidden_size, len(tag_vocab), opt.mlp_dropout, opt.C_bn) else: C = SpMlpTagger(opt.C_layers, opt.shared_hidden_size, opt.private_hidden_size, opt.concat_sp, opt.shared_hidden_size + opt.private_hidden_size, len(tag_vocab), opt.mlp_dropout, opt.C_bn) if opt.shared_hidden_size > 0 and opt.n_critic > 0: if opt.D_model.lower() == 'lstm': d_args = { 'num_layers': opt.D_lstm_layers, 'input_size': opt.shared_hidden_size, 'hidden_size': opt.shared_hidden_size, 'word_dropout': opt.D_word_dropout, 'dropout': opt.D_dropout, 'bdrnn': opt.D_bdrnn, 'attn_type': opt.D_attn } elif opt.D_model.lower() == 'cnn': d_args = { 'num_layers': 1, 'input_size': opt.shared_hidden_size, 'hidden_size': opt.shared_hidden_size, 'kernel_num': opt.D_kernel_num, 'kernel_sizes': opt.D_kernel_sizes, 'word_dropout': opt.D_word_dropout, 'dropout': opt.D_dropout } else: d_args = None if opt.D_model.lower() == 'mlp': D = MLPLanguageDiscriminator(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_langs), opt.loss, opt.D_dropout, opt.D_bn) else: D = LanguageDiscriminator(opt.D_model, opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_langs), opt.D_dropout, opt.D_bn, d_args) if opt.use_data_parallel: F_s, C, D = nn.DataParallel(F_s).to(opt.device) if F_s else None, nn.DataParallel(C).to(opt.device), nn.DataParallel(D).to(opt.device) if D else None else: F_s, C, D = F_s.to(opt.device) if F_s else None, C.to(opt.device), D.to(opt.device) if D else None if F_p: if opt.use_data_parallel: F_p = nn.DataParallel(F_p).to(opt.device) else: F_p = F_p.to(opt.device) # optimizers optimizer = optim.Adam(filter(lambda p: p.requires_grad, itertools.chain(*map(list, [emb.parameters(), F_s.parameters() if F_s else [], \ C.parameters(), F_p.parameters() if F_p else []]))), lr=opt.learning_rate, weight_decay=opt.weight_decay) if D: optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate, weight_decay=opt.D_weight_decay) # testing if opt.test_only: log.info(f'Loading model from {opt.model_save_file}...') if F_s: F_s.load_state_dict(torch.load(os.path.join(opt.model_save_file, f'netF_s.pth'))) for lang in opt.all_langs: F_p.load_state_dict(torch.load(os.path.join(opt.model_save_file, f'net_F_p.pth'))) C.load_state_dict(torch.load(os.path.join(opt.model_save_file, f'netC.pth'))) if D: D.load_state_dict(torch.load(os.path.join(opt.model_save_file, f'netD.pth'))) log.info('Evaluating validation sets:') acc = {} log.info(dev_loaders) log.info(vocabs) for lang in opt.all_langs: acc[lang] = evaluate(f'{lang}_dev', dev_loaders[lang], vocabs[lang], tag_vocab, emb, lang, F_s, F_p, C) avg_acc = sum([acc[d] for d in opt.dev_langs]) / len(opt.dev_langs) log.info(f'Average validation accuracy: {avg_acc}') log.info('Evaluating test sets:') test_acc = {} for lang in opt.all_langs: test_acc[lang] = evaluate(f'{lang}_test', test_loaders[lang], vocabs[lang], tag_vocab, emb, lang, F_s, F_p, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_langs]) / len(opt.dev_langs) log.info(f'Average test accuracy: {avg_test_acc}') return {'valid': acc, 'test': test_acc} # training best_acc, best_avg_acc = defaultdict(float), 0.0 epochs_since_decay = 0 # lambda scheduling if opt.lambd > 0 and opt.lambd_schedule: opt.lambd_orig = opt.lambd num_iter = int(utils.gmean([len(train_loaders[l]) for l in opt.langs])) # adapt max_epoch if opt.max_epoch > 0 and num_iter * opt.max_epoch < 15000: opt.max_epoch = 15000 // num_iter log.info(f"Setting max_epoch to {opt.max_epoch}") for epoch in range(opt.max_epoch): emb.train() if F_s: F_s.train() C.train() if D: D.train() if F_p: F_p.train() # lambda scheduling if hasattr(opt, 'lambd_orig') and opt.lambd_schedule: if epoch == 0: opt.lambd = opt.lambd_orig elif epoch == 5: opt.lambd = 10 * opt.lambd_orig elif epoch == 15: opt.lambd = 100 * opt.lambd_orig log.info(f'Scheduling lambda = {opt.lambd}') # training accuracy correct, total = defaultdict(int), defaultdict(int) gate_correct = defaultdict(int) c_gate_correct = defaultdict(int) # D accuracy d_correct, d_total = 0, 0 for i in tqdm(range(num_iter), ascii=True): # D iterations if opt.shared_hidden_size > 0: utils.freeze_net(emb) utils.freeze_net(F_s) utils.freeze_net(F_p) utils.freeze_net(C) utils.unfreeze_net(D) # WGAN n_critic trick since D trains slower n_critic = opt.n_critic if opt.wgan_trick: if opt.n_critic>0 and ((epoch==0 and i<25) or i%500==0): n_critic = 100 for _ in range(n_critic): D.zero_grad() loss_d = {} lang_features = {} # train on both labeled and unlabeled langs for lang in opt.all_langs: # targets not used d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, d_unlabeled_iters, lang) d_inputs, d_lengths, mask, d_chars, d_char_lengths = d_inputs d_embeds = emb(lang, d_inputs, d_chars, d_char_lengths) shared_feat = F_s((d_embeds, d_lengths)) if opt.grad_penalty != 'none': lang_features[lang] = shared_feat.detach() if opt.D_model.lower() == 'mlp': d_outputs = D(shared_feat) # if token-level D, we can reuse the gate label generator d_targets = utils.get_gate_label(d_outputs, lang, mask, False, all_langs=True) d_total += torch.sum(d_lengths).item() else: d_outputs = D((shared_feat, d_lengths)) d_targets = utils.get_lang_label(opt.loss, lang, len(d_lengths)) d_total += len(d_lengths) # D accuracy _, pred = torch.max(d_outputs, -1) # d_total += len(d_lengths) d_correct += (pred==d_targets).sum().item() if opt.use_data_parallel: l_d = functional.nll_loss(d_outputs.view(-1, D.module.num_langs), d_targets.view(-1), ignore_index=-1) else: l_d = functional.nll_loss(d_outputs.view(-1, D.num_langs), d_targets.view(-1), ignore_index=-1) l_d.backward() loss_d[lang] = l_d.item() # gradient penalty if opt.grad_penalty != 'none': gp = utils.calc_gradient_penalty(D, lang_features, onesided=opt.onesided_gp, interpolate=(opt.grad_penalty=='wgan')) gp.backward() optimizerD.step() # F&C iteration utils.unfreeze_net(emb) if opt.use_wordemb and opt.fix_emb: for lang in emb.langs: emb.wordembs[lang].weight.requires_grad = False if opt.use_charemb and opt.fix_charemb: emb.charemb.weight.requires_grad = False utils.unfreeze_net(F_s) utils.unfreeze_net(F_p) utils.unfreeze_net(C) utils.freeze_net(D) emb.zero_grad() if F_s: F_s.zero_grad() if F_p: F_p.zero_grad() C.zero_grad() # optimizer.zero_grad() for lang in opt.langs: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, lang) inputs, lengths, mask, chars, char_lengths = inputs bs, seq_len = inputs.size() embeds = emb(lang, inputs, chars, char_lengths) shared_feat, private_feat = None, None if opt.shared_hidden_size > 0: shared_feat = F_s((embeds, lengths)) if opt.private_hidden_size > 0: private_feat, gate_outputs = F_p((embeds, lengths)) if opt.C_MoE: c_outputs, c_gate_outputs = C((shared_feat, private_feat)) else: c_outputs = C((shared_feat, private_feat)) # targets are padded with -1 l_c = functional.nll_loss(c_outputs.view(bs*seq_len, -1), targets.view(-1), ignore_index=-1) # gate loss if F_p: gate_targets = utils.get_gate_label(gate_outputs, lang, mask, False) l_gate = functional.cross_entropy(gate_outputs.view(bs*seq_len, -1), gate_targets.view(-1), ignore_index=-1) l_c += opt.gate_loss_weight * l_gate _, gate_pred = torch.max(gate_outputs.view(bs*seq_len, -1), -1) gate_correct[lang] += (gate_pred == gate_targets.view(-1)).sum().item() if opt.C_MoE and opt.C_gate_loss_weight > 0: c_gate_targets = utils.get_gate_label(c_gate_outputs, lang, mask, opt.expert_sp) _, c_gate_pred = torch.max(c_gate_outputs.view(bs*seq_len, -1), -1) if opt.expert_sp: l_c_gate = functional.binary_cross_entropy_with_logits( mask.unsqueeze(-1) * c_gate_outputs, c_gate_targets) c_gate_correct[lang] += torch.index_select(c_gate_targets.view(bs*seq_len, -1), -1, c_gate_pred.view(bs*seq_len)).sum().item() else: l_c_gate = functional.cross_entropy(c_gate_outputs.view(bs*seq_len, -1), c_gate_targets.view(-1), ignore_index=-1) c_gate_correct[lang] += (c_gate_pred == c_gate_targets.view(-1)).sum().item() l_c += opt.C_gate_loss_weight * l_c_gate l_c.backward() _, pred = torch.max(c_outputs, -1) total[lang] += torch.sum(lengths).item() correct[lang] += (pred == targets).sum().item() # update F with D gradients on all langs if D: for lang in opt.all_langs: inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, lang) inputs, lengths, mask, chars, char_lengths = inputs embeds = emb(lang, inputs, chars, char_lengths) shared_feat = F_s((embeds, lengths)) # d_outputs = D((shared_feat, lengths)) if opt.D_model.lower() == 'mlp': d_outputs = D(shared_feat) # if token-level D, we can reuse the gate label generator d_targets = utils.get_gate_label(d_outputs, lang, mask, False, all_langs=True) else: d_outputs = D((shared_feat, lengths)) d_targets = utils.get_lang_label(opt.loss, lang, len(lengths)) if opt.use_data_parallel: l_d = functional.nll_loss(d_outputs.view(-1, D.module.num_langs), d_targets.view(-1), ignore_index=-1) else: l_d = functional.nll_loss(d_outputs.view(-1, D.num_langs), d_targets.view(-1), ignore_index=-1) if opt.lambd > 0: l_d *= -opt.lambd l_d.backward() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch+1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0*d_correct/d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.langs)) log.info('\t'.join([str(100.0*correct[d]/total[d]) for d in opt.langs])) log.info('Gate accuracy:') log.info('\t'.join([str(100.0*gate_correct[d]/total[d]) for d in opt.langs])) log.info('Tagger Gate accuracy:') log.info('\t'.join([str(100.0*c_gate_correct[d]/total[d]) for d in opt.langs])) log.info('Evaluating validation sets:') acc = {} for lang in opt.dev_langs: acc[lang] = evaluate(f'{lang}_dev', dev_loaders[lang], vocabs[lang], tag_vocab, emb, lang, F_s, F_p, C) avg_acc = sum([acc[d] for d in opt.dev_langs]) / len(opt.dev_langs) log.info(f'Average validation accuracy: {avg_acc}') log.info('Evaluating test sets:') test_acc = {} for lang in opt.dev_langs: test_acc[lang] = evaluate(f'{lang}_test', test_loaders[lang], vocabs[lang], tag_vocab, emb, lang, F_s, F_p, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_langs]) / len(opt.dev_langs) log.info(f'Average test accuracy: {avg_test_acc}') if avg_acc > best_avg_acc: epochs_since_decay = 0 log.info(f'New best average validation accuracy: {avg_acc}') best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) if F_s: torch.save(F_s.state_dict(), '{}/netF_s.pth'.format(opt.model_save_file)) torch.save(emb.state_dict(), '{}/net_emb.pth'.format(opt.model_save_file)) if F_p: torch.save(F_p.state_dict(), '{}/net_F_p.pth'.format(opt.model_save_file)) torch.save(C.state_dict(), '{}/netC.pth'.format(opt.model_save_file)) if D: torch.save(D.state_dict(), '{}/netD.pth'.format(opt.model_save_file)) else: epochs_since_decay += 1 if opt.lr_decay < 1 and epochs_since_decay >= opt.lr_decay_epochs: epochs_since_decay = 0 old_lr = optimizer.param_groups[0]['lr'] optimizer.param_groups[0]['lr'] = old_lr * opt.lr_decay log.info(f'Decreasing LR to {old_lr * opt.lr_decay}') # end of training log.info(f'Best average validation accuracy: {best_avg_acc}') return best_acc
def train(target_domain_idx, train_sets, test_sets): """ train_sets, test_sets: unlabeled domain(dev domain) -> AmazonDataset """ # test dataset print(train_sets[0]) print(test_sets[0]) print(train_sets[0][0].shape) print(test_sets[0][1].shape) # ---------------------- dataloader ---------------------- # train_loaders = DataLoader(train_sets, opt.batch_size, shuffle=False) test_loaders = DataLoader(test_sets, opt.batch_size, shuffle=False) # ---------------------- model initialization ---------------------- # F_s = None F_d = {} C = None if opt.model.lower() == 'mlp': F_s = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.shared_hidden_size, opt.dropout, opt.F_bn) for domain in opt.domains: F_d[domain] = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.domain_hidden_size, opt.dropout, opt.F_bn) else: raise Exception(f'Unknown model architecture {opt.model}') C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn) # 转移到gpu上 F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) # ---------------------- load pre-training model ---------------------- # log.info(f'Loading model from {opt.exp2_model_save_file}...') F_s.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file, f'netF_s.pth'))) for domain in opt.all_domains: if domain in F_d: F_d[domain].load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file, f'net_F_d_{domain}.pth'))) C.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file, f'netC.pth'))) D.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file, f'netD.pth'))) # ---------------------- get fake label(hard label) ---------------------- # log.info('Get fake label:') pseudo_labels, _ = genarate_labels(target_domain_idx, False, train_loaders, F_s, F_d, C, D) # ********************** Test the accuracy of the label prediction ********************** # test_pseudo_labels, targets_total = genarate_labels(target_domain_idx, True, test_loaders, F_s, F_d, C, D) label_correct_acc = calc_label_prediction(test_pseudo_labels, targets_total) log.info(f'the correct rate of label prediction: {label_correct_acc}') # ********************** Test the accuracy of the label prediction ********************** # # ------------------------------- 构造target数据集 ------------------------------- # target_dataset = Subset(train_sets, pseudo_labels) target_dataloader_labelled = DataLoader(target_dataset, opt.batch_size, shuffle=True) target_train_iters = iter(target_dataloader_labelled) # ------------------------------- F_d_target模型以及一些必要的参数定义 ------------------------------- # F_d_target = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.domain_hidden_size, opt.dropout, opt.F_bn) F_d_target = F_d_target.to(opt.device) # ******************************* 加载预训练模型的权重,不用再从头训练 ******************************* # f_d_choice = random.randint(0, len(opt.domains)) F_d_target.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file, f'net_F_d_{opt.domains[f_d_choice]}.pth'))) optimizer_F_d_target = optim.Adam(F_d_target.parameters(), lr=opt.learning_rate) # ------------------------------- 测试一下只利用shared feature的准确率 ------------------------------- # log.info('Evaluating test sets only on shared feature:') test_acc = evaluate(opt.dev_domains, test_loaders, F_s, None, C) log.info(f'test accuracy: {test_acc}') for epoch in range(opt.max_epoch): F_d_target.train() C.train() # training accuracy correct, total = 0, 0 num_iter = len(target_dataloader_labelled) print(num_iter) for _ in tqdm(range(num_iter)): utils.freeze_net(F_s) C.zero_grad() F_d_target.zero_grad() inputs, targets = utils.endless_get_next_batch(target_dataloader_labelled, target_train_iters) targets = targets.to(opt.device) shared_feat = F_s(inputs) domain_feat = F_d_target(inputs) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward() _, pred_idx = torch.max(c_outputs, 1) total += targets.size(0) correct += (pred_idx == targets).sum().item() optimizer_F_d_target.step() # 暂时留下一个问题,这里没有写验证集,因为如果这里要设置验证集的话,需要将训练集切割 # 因为unlabeled domain里面只有2000个label sample以及raw_unlabeled_sets(四千多张) # 后者训练时用,前者测试的时候用 # end of epoch print("correct is %d" % correct) print("total is %d" % total) log.info('Ending epoch {}'.format(epoch + 1)) log.info('Training accuracy:') log.info(opt.unlabeled_domains) log.info(str(100.0 * correct / total)) # 训练过程中的验证集 print("correct is %d" % correct) print("total is %d" % total) log.info('Ending epoch {}'.format(epoch + 1)) log.info('Training accuracy:') log.info('\t'.join(opt.unlabeled_domains)) log.info(str(100.0 * correct / total)) # 保存模型 torch.save(F_d_target.state_dict(), '{}/net_F_d_target.pth'.format(opt.exp2_target_model_save_file)) torch.save(C.state_dict(), '{}/netC_target.pth'.format(opt.exp2_target_model_save_file)) # 在测试集上测试,选择2000张有label的标签数据 # 方便对比与单独训练有label时的情况 log.info('Evaluating test sets:') test_acc = evaluate(opt.dev_domains, test_loaders, F_s, F_d_target, C) log.info(f'test accuracy: {test_acc}') return test_acc
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets): """ train_sets, dev_sets, test_sets: dict[domain] -> AmazonDataset For unlabeled domains, no train_sets are available """ # dataset loaders train_loaders, unlabeled_loaders = {}, {} train_iters, unlabeled_iters = {}, {} dev_loaders, test_loaders = {}, {} my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate for domain in opt.domains: train_loaders[domain] = DataLoader(train_sets[domain], opt.batch_size, shuffle=True, collate_fn=my_collate) train_iters[domain] = iter(train_loaders[domain]) for domain in opt.dev_domains: dev_loaders[domain] = DataLoader(dev_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) test_loaders[domain] = DataLoader(test_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) for domain in opt.all_domains: if domain in opt.unlabeled_domains: uset = unlabeled_sets[domain] else: # for labeled domains, consider which data to use as unlabeled set if opt.unlabeled_data == 'both': uset = ConcatDataset( [train_sets[domain], unlabeled_sets[domain]]) elif opt.unlabeled_data == 'unlabeled': uset = unlabeled_sets[domain] elif opt.unlabeled_data == 'train': uset = train_sets[domain] else: raise Exception( f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}' ) unlabeled_loaders[domain] = DataLoader(uset, opt.batch_size, shuffle=True, collate_fn=my_collate) unlabeled_iters[domain] = iter(unlabeled_loaders[domain]) # model F_s = None F_d = {} C, D = None, None if opt.model.lower() == 'dan': F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) for domain in opt.domains: F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) elif opt.model.lower() == 'lstm': F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.dropout, opt.bdrnn, opt.attn) for domain in opt.domains: F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.dropout, opt.bdrnn, opt.attn) elif opt.model.lower() == 'cnn': F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) for domain in opt.domains: F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) else: raise Exception(f'Unknown model architecture {opt.model}') C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn) F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain( *map(list, [F_s.parameters() if F_s else [], C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate) optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate) # testing if opt.test_only: log.info(f'Loading model from {opt.exp3_model_save_file}...') if F_s: F_s.load_state_dict( torch.load( os.path.join(opt.exp3_model_save_file, f'netF_s.pth'))) for domain in opt.all_domains: if domain in F_d: F_d[domain].load_state_dict( torch.load( os.path.join(opt.exp3_model_save_file, f'net_F_d_{domain}.pth'))) C.load_state_dict( torch.load(os.path.join(opt.exp3_model_save_file, f'netC.pth'))) D.load_state_dict( torch.load(os.path.join(opt.exp3_model_save_file, f'netD.pth'))) log.info('Evaluating validation sets:') acc = {} for domain in opt.all_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average validation accuracy: {avg_acc}') log.info('Evaluating test sets:') test_acc = {} for domain in opt.all_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average test accuracy: {avg_test_acc}') return {'valid': acc, 'test': test_acc} # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() D.train() LAMBDA = 3 lambda1 = 0.1 lambda2 = 0.1 for f in F_d.values(): f.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy shared_d_correct, private_d_correct, d_total = 0, 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # D iterations utils.freeze_net(F_s) map(utils.freeze_net, F_d.values()) utils.freeze_net(C) utils.unfreeze_net(D) # WGAN n_critic trick since D trains slower # ********************** D iterations on all domains ********************** # # ---------------------- update D with D gradients on all domains ---------------------- # n_critic = opt.n_critic if opt.wgan_trick: if opt.n_critic > 0 and ((epoch == 0 and i < 25) or i % 500 == 0): n_critic = 100 for _ in range(n_critic): D.zero_grad() loss_d = {} # train on both labeled and unlabeled domains for domain in opt.all_domains: # targets not usedndless_get_next_batch( # unlabeled_loaders, unlabeled_iters, domain) d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1])) shared_feat = F_s(d_inputs) shared_d_outputs = D(shared_feat) _, shared_pred = torch.max(shared_d_outputs, 1) if domain != opt.dev_domains[0]: private_feat = F_d[domain](d_inputs) private_d_outputs = D(private_feat) _, private_pred = torch.max(private_d_outputs, 1) d_total += len(d_inputs[1]) if opt.loss.lower() == 'l2': _, tgt_indices = torch.max(d_targets, 1) shared_d_correct += ( shared_pred == tgt_indices).sum().item() shared_l_d = functional.mse_loss( shared_d_outputs, d_targets) private_l_d = 0.0 if domain != opt.dev_domains[0]: private_d_correct += ( private_pred == tgt_indices).sum().item() private_l_d = functional.mse_loss( private_d_outputs, d_targets) / len( opt.domains) l_d_sum = shared_l_d + private_l_d l_d_sum.backward() else: shared_d_correct += ( shared_pred == d_targets).sum().item() shared_l_d = functional.nll_loss( shared_d_outputs, d_targets) private_l_d = 0.0 if domain != opt.dev_domains[0]: private_d_correct += ( private_pred == d_targets).sum().item() private_l_d = functional.nll_loss( private_d_outputs, d_targets) / len( opt.domains) l_d_sum = shared_l_d + private_l_d l_d_sum.backward() loss_d[domain] = l_d_sum.item() optimizerD.step() # ---------------------- update D with C gradients on all domains ---------------------- # # ********************** D iterations on all domains ********************** # # ********************** F&C iteration ********************** # # ---------------------- update F_s & F_ds with C gradients on all labeled domains ---------------------- # # F&C iteration utils.unfreeze_net(F_s) map(utils.unfreeze_net, F_d.values()) utils.unfreeze_net(C) utils.freeze_net(D) if opt.fix_emb: utils.freeze_net(F_s.word_emb) for f_d in F_d.values(): utils.freeze_net(f_d.word_emb) F_s.zero_grad() for f_d in F_d.values(): f_d.zero_grad() C.zero_grad() for domain in opt.domains: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, domain) targets = targets.to(opt.device) shared_feat = F_s(inputs) domain_feat = F_d[domain](inputs) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) loss_part_1 = functional.nll_loss(c_outputs, targets) targets = targets.unsqueeze(1) targets_onehot = torch.FloatTensor(opt.batch_size, 2) targets_onehot.zero_() targets_onehot.scatter_(1, targets.cpu(), 1) targets_onehot = targets_onehot.to(opt.device) loss_part_2 = lambda1 * margin_regularization( inputs, targets_onehot, F_d[domain], LAMBDA) loss_part_3 = -lambda2 * center_point_constraint( domain_feat, targets) print("lambda1: " + str(lambda1)) print("lambda2: " + str(lambda2)) print("loss_part_1: " + str(loss_part_1)) print("loss_part_2: " + str(loss_part_2)) print("loss_part_3: " + str(loss_part_3)) l_c = loss_part_1 + loss_part_2 + loss_part_3 l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() # ---------------------- update F_s & F_ds with C gradients on all labeled domains ---------------------- # # ---------------------- update F_s with D gradients on all domains ---------------------- # # update F with D gradients on all domains for domain in opt.all_domains: d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) shared_feat = F_s(d_inputs) shared_d_outputs = D(shared_feat) if domain != opt.dev_domains[0]: private_feat = F_d[domain](d_inputs) private_d_outputs = D(private_feat) l_d_sum = None if opt.loss.lower() == 'gr': d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1])) shared_l_d = functional.nll_loss(shared_d_outputs, d_targets) private_l_d, l_d_sum = 0.0, 0.0 if domain != opt.dev_domains[0]: # 注意这边的loss function private_l_d = functional.nll_loss( private_d_outputs, d_targets) * -1. / len( opt.domains) if opt.shared_lambd > 0: l_d_sum = shared_l_d * opt.shared_lambd * -1. else: l_d_sum = shared_l_d * -1. if opt.private_lambd > 0: l_d_sum += private_l_d * opt.private_lambd * -1. else: l_d_sum += private_l_d * -1. elif opt.loss.lower() == 'bs': d_targets = utils.get_random_domain_label( opt.loss, len(d_inputs[1])) shared_l_d = functional.kl_div(shared_d_outputs, d_targets, size_average=False) private_l_d, l_d_sum = 0.0, 0.0 if domain != opt.dev_domains[0]: private_l_d = functional.kl_div(private_d_outputs, d_targets, size_average=False) \ * -1. / len(opt.domains) if opt.shared_lambd > 0: l_d_sum = shared_l_d * opt.shared_lambd else: l_d_sum = shared_l_d if opt.private_lambd > 0: l_d_sum += private_l_d * opt.private_lambd else: l_d_sum += private_l_d elif opt.loss.lower() == 'l2': d_targets = utils.get_random_domain_label( opt.loss, len(d_inputs[1])) shared_l_d = functional.mse_loss(shared_d_outputs, d_targets) private_l_d, l_d_sum = 0.0, 0.0 if domain != opt.dev_domains[0]: private_l_d = functional.mse_loss( private_d_outputs, d_targets) * -1. / len( opt.domains) if opt.shared_lambd > 0: l_d_sum = shared_l_d * opt.shared_lambd else: l_d_sum = shared_l_d if opt.private_lambd > 0: l_d_sum += private_l_d * opt.private_lambd else: l_d_sum += private_l_d l_d_sum.backward() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('shared D Training Accuracy: {}%'.format( 100.0 * shared_d_correct / d_total)) log.info('private D Training Accuracy(average): {}%'.format( 100.0 * private_d_correct / len(opt.domains) / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join( [str(100.0 * correct[d] / total[d]) for d in opt.domains])) # 验证集上验证实验 log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average validation accuracy: {avg_acc}') # 测试集上验证实验 log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average test accuracy: {avg_test_acc}') # 保存模型 if avg_acc > best_avg_acc: log.info(f'New best average validation accuracy: {avg_acc}') best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.exp3_model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) torch.save(F_s.state_dict(), '{}/netF_s.pth'.format(opt.exp3_model_save_file)) for d in opt.domains: if d in F_d: torch.save( F_d[d].state_dict(), '{}/net_F_d_{}.pth'.format(opt.exp3_model_save_file, d)) torch.save(C.state_dict(), '{}/netC.pth'.format(opt.exp3_model_save_file)) torch.save(D.state_dict(), '{}/netD.pth'.format(opt.exp3_model_save_file)) # end of training log.info(f'Best average validation accuracy: {best_avg_acc}') log.info( f'Loading model for feature visualization from {opt.exp3_model_save_file}...' ) for domain in opt.domains: F_d[domain].load_state_dict( torch.load( os.path.join(opt.exp3_model_save_file, f'net_F_d_{domain}.pth'))) num_iter = len(train_loaders[opt.domains[0]]) # visual_features暂时不加上shared feature # visual_features, senti_labels = get_visual_features(num_iter, unlabeled_loaders, unlabeled_iters, F_s, F_d) visual_features, senti_labels = get_visual_features( num_iter, unlabeled_loaders, unlabeled_iters, F_d) return best_acc, visual_features, senti_labels
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets): train_loaders, unlabeled_loaders = {}, {} train_iters, unlabeled_iters = {}, {} dev_loaders, test_loaders = {}, {} my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate for domain in opt.domains: train_loaders[domain] = DataLoader(train_sets[domain], opt.batch_size, shuffle=True, collate_fn=my_collate) train_iters[domain] = iter(train_loaders[domain]) for domain in opt.dev_domains: dev_loaders[domain] = DataLoader(dev_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) test_loaders[domain] = DataLoader(test_sets[domain], opt.batch_size, shuffle=False, collate_fn=my_collate) for domain in opt.all_domains: if domain in opt.unlabeled_domains: uset = unlabeled_sets[domain] else: # for labeled domains, consider which data to use as unlabeled set if opt.unlabeled_data == 'both': uset = ConcatDataset( [train_sets[domain], unlabeled_sets[domain]]) elif opt.unlabeled_data == 'unlabeled': uset = unlabeled_sets[domain] elif opt.unlabeled_data == 'train': uset = train_sets[domain] else: raise Exception( 'Unknown options for the unlabeled data usage: {}'.format( opt.unlabeled_data)) unlabeled_loaders[domain] = DataLoader(uset, opt.batch_size, shuffle=True, collate_fn=my_collate) unlabeled_iters[domain] = iter(unlabeled_loaders[domain]) # models F_s = None C = None if opt.model.lower() == 'dan': F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) elif opt.model.lower() == 'lstm': F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.dropout, opt.bdrnn, opt.attn) elif opt.model.lower() == 'cnn': F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) else: raise Exception('Unknown model architecture {}'.format(opt.model)) C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) F_s, C = F_s.to(opt.device), C.to(opt.device) optimizer = optim.Adam(itertools.chain( *map(list, [F_s.parameters() if F_s else [], C.parameters()])), lr=opt.learning_rate) best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # F&C iteration utils.unfreeze_net(F_s) utils.unfreeze_net(C) if opt.fix_emb: utils.freeze_net(F_s.word_emb) F_s.zero_grad() C.zero_grad() for domain in opt.domains: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, domain) targets = targets.to(opt.device) shared_feat = F_s(inputs) domain_feat = torch.zeros( len(targets), opt.domain_hidden_size).to(opt.device) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join( [str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) if avg_acc > best_avg_acc: log.info( 'New best average validation accuracy: {}'.format(avg_acc)) best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) torch.save(F_s.state_dict(), '{}/netF_s.pth'.format(opt.model_save_file)) torch.save(C.state_dict(), '{}/netC.pth'.format(opt.model_save_file)) # end of training log.info('Best average validation accuracy: {}'.format(best_avg_acc)) return best_acc
def train(train_sets, test_sets): # ---------------------- dataloader ---------------------- # # dataset loaders train_loaders, train_iters, test_loaders, test_iters = {}, {}, {}, {} # 加载有label的训练数据 for domain in opt.domains: train_loaders[domain] = DataLoader(train_sets[domain], opt.batch_size, shuffle=True) train_iters[domain] = iter(train_loaders[domain]) test_loaders[domain] = DataLoader(test_sets[domain], opt.batch_size, shuffle=False) test_iters[domain] = iter(test_loaders[domain]) # ---------------------- model initialization ---------------------- # F_d = {} C = None if opt.model.lower() == 'mlp': for domain in opt.domains: F_d[domain] = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.domain_hidden_size, opt.dropout, opt.F_bn) C = SentimentClassifier(opt.C_layers, opt.domain_hidden_size, opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) # 转移到gpu上 C = C.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) criterion_cent = CenterLoss(num_classes=2, feat_dim=opt.domain_hidden_size, use_gpu=True) optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=0.5) optimizer = optim.Adam(itertools.chain( *map(list, [C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate) # training correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 best_acc = 0.0 best_acc_dict = {} margin = 3 margin_lambda = 0.1 # center_loss_weight_cent = 0.1 for epoch in range(opt.max_epoch): C.train() for f in F_d.values(): f.train() # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) # First_stage for _ in tqdm(range(num_iter)): for f_d in F_d.values(): f_d.zero_grad() C.zero_grad() optimizer_centloss.zero_grad() for domain in opt.domains: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, domain) targets = targets.to(opt.device) domain_feat = F_d[domain](inputs) visual_feature, c_outputs = C(domain_feat) # loss_cent = criterion_cent(visual_feature, targets) # loss_cent *= center_loss_weight_cent loss_cent = 0.0 loss_part_1 = functional.nll_loss(c_outputs, targets) targets = targets.unsqueeze(1) targets_onehot = torch.FloatTensor(opt.batch_size, 2) targets_onehot.zero_() targets_onehot.scatter_(1, targets.cpu(), 1) targets_onehot = targets_onehot.to(opt.device) loss_part_2 = margin_lambda * margin_regularization( inputs, targets_onehot, F_d[domain], C, margin) # loss_part_2 = 0.0 print("loss_part_1: " + str(loss_part_1)) print("loss_part_2: " + str(loss_part_2)) print("loss_cent: " + str(loss_cent)) l_c = loss_part_1 + loss_part_2 + loss_cent l_c.backward() _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() optimizer.step() # for param in criterion_cent.parameters(): # param.grad.data *= (1. / center_loss_weight_cent) optimizer_centloss.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join( [str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating test sets:') test_acc = {} for domain in opt.domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_d[domain], C) avg_test_acc = sum([test_acc[d] for d in opt.domains]) / len(opt.domains) log.info(f'Average test accuracy: {avg_test_acc}') if avg_test_acc > best_acc: log.info(f'New best Average test accuracy: {avg_test_acc}') best_acc = avg_test_acc best_acc_dict = test_acc for d in opt.domains: if d in F_d: torch.save( F_d[d].state_dict(), '{}/net_F_d_{}.pth'.format(opt.exp2_model_save_file, d)) torch.save(C.state_dict(), '{}/netC.pth'.format(opt.exp2_model_save_file)) log.info( f'Loading model for feature visualization from {opt.exp2_model_save_file}...' ) for domain in opt.domains: F_d[domain].load_state_dict( torch.load( os.path.join(opt.exp2_model_save_file, f'net_F_d_{domain}.pth'))) num_iter = len(train_loaders[opt.domains[0]]) visual_features, senti_labels = get_visual_features( num_iter, test_loaders, test_iters, F_d, C) # visual_features, senti_labels = get_visual_features(num_iter, train_loaders, train_iters, F_d) return best_acc, best_acc_dict, visual_features, senti_labels
def train_nobert(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders): # models F_s = None F_d = {} C, D = None, None if opt.model.lower() == 'dan': F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) for domain in opt.domains: F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) elif opt.model.lower() == 'lstm': F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.dropout, opt.bdrnn, opt.attn) for domain in opt.domains: F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.dropout, opt.bdrnn, opt.attn) elif opt.model.lower() == 'cnn': F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) for domain in opt.domains: F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) else: raise Exception('Unknown model architecture {}'.format(opt.model)) C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn) F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain(*map(list, [F_s.parameters() if F_s else [], C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate) optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate) # testing if opt.test_only: log.info('Loading model from {}...'.format(opt.model_save_file)) if F_s: F_s.load_state_dict(torch.load(os.path.join(opt.model_save_file, 'netF_s.pth'))) for domain in opt.all_domains: if domain in F_d: F_d[domain].load_state_dict(torch.load(os.path.join(opt.model_save_file, 'net_F_d_{}.pth'.format(domain)))) C.load_state_dict(torch.load(os.path.join(opt.model_save_file, 'netC.pth'))) D.load_state_dict(torch.load(os.path.join(opt.model_save_file, 'netD.pth'))) log.info('Evaluating validation sets:') acc = {} for domain in opt.all_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.all_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) return {'valid': acc, 'test': test_acc} # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() D.train() for f in F_d.values(): f.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # D iterations utils.freeze_net(F_s) map(utils.freeze_net, F_d.values()) utils.freeze_net(C) utils.unfreeze_net(D) # WGAN n_critic trick since D trains slower n_critic = opt.n_critic if opt.wgan_trick: if opt.n_critic>0 and ((epoch==0 and i<25) or i%500==0): n_critic = 100 for _ in range(n_critic): D.zero_grad() loss_d = {} # train on both labeled and unlabeled domains for domain in opt.all_domains: # targets not used batch = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) batch = tuple(t.to(opt.device) for t in batch) emb_ids, input_ids, input_mask, segment_ids, label_ids = batch d_targets = utils.get_domain_label(opt.loss, domain, len(emb_ids)) shared_feat = F_s(emb_ids) d_outputs = D(shared_feat) # D accuracy _, pred = torch.max(d_outputs, 1) d_total += len(emb_ids) if opt.loss.lower() == 'l2': _, tgt_indices = torch.max(d_targets, 1) d_correct += (pred==tgt_indices).sum().item() l_d = functional.mse_loss(d_outputs, d_targets) l_d.backward() else: d_correct += (pred==d_targets).sum().item() l_d = functional.nll_loss(d_outputs, d_targets) l_d.backward() loss_d[domain] = l_d.item() optimizerD.step() # F&C iteration utils.unfreeze_net(F_s) map(utils.unfreeze_net, F_d.values()) utils.unfreeze_net(C) utils.freeze_net(D) # if opt.fix_emb: # utils.freeze_net(F_s.word_emb) # for f_d in F_d.values(): # utils.freeze_net(f_d.word_emb) F_s.zero_grad() for f_d in F_d.values(): f_d.zero_grad() C.zero_grad() for domain in opt.domains: batch = utils.endless_get_next_batch( train_loaders, train_iters, domain) batch = tuple(t.to(opt.device) for t in batch) emb_ids, input_ids, input_mask, segment_ids, label_ids = batch inputs = emb_ids targets = label_ids shared_feat = F_s(inputs) domain_feat = F_d[domain](inputs) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() # update F with D gradients on all domains for domain in opt.all_domains: batch = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) batch = tuple(t.to(opt.device) for t in batch) emb_ids, input_ids, input_mask, segment_ids, label_ids = batch d_inputs = emb_ids shared_feat = F_s(d_inputs) d_outputs = D(shared_feat) if opt.loss.lower() == 'gr': d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs)) l_d = functional.nll_loss(d_outputs, d_targets) if opt.lambd > 0: l_d *= -opt.lambd elif opt.loss.lower() == 'bs': d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs)) l_d = functional.kl_div(d_outputs, d_targets, size_average=False) if opt.lambd > 0: l_d *= opt.lambd elif opt.loss.lower() == 'l2': d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs)) l_d = functional.mse_loss(d_outputs, d_targets) if opt.lambd > 0: l_d *= opt.lambd l_d.backward() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch+1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0*d_correct/d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join([str(100.0*correct[d]/total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) if avg_acc > best_avg_acc: log.info('New best average validation accuracy: {}'.format(avg_acc)) best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) # torch.save(F_s.state_dict(), # '{}/netF_s.pth'.format(opt.model_save_file)) # for d in opt.domains: # if d in F_d: # torch.save(F_d[d].state_dict(), # '{}/net_F_d_{}.pth'.format(opt.model_save_file, d)) # torch.save(C.state_dict(), # '{}/netC.pth'.format(opt.model_save_file)) # torch.save(D.state_dict(), # '{}/netD.pth'.format(opt.model_save_file)) # end of training log.info('Best average validation accuracy: {}'.format(best_avg_acc)) return best_acc
def train_private_nobert(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders): # models F_d = {} C = None if opt.model.lower() == 'dan': for domain in opt.domains: F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.sum_pooling, opt.dropout, opt.F_bn) elif opt.model.lower() == 'lstm': for domain in opt.domains: F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.dropout, opt.bdrnn, opt.attn) elif opt.model.lower() == 'cnn': for domain in opt.domains: F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size, opt.kernel_num, opt.kernel_sizes, opt.dropout) else: raise Exception('Unknown model architecture {}'.format(opt.model)) C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) C = C.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain( *map(list, [[], C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate) # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): C.train() for f in F_d.values(): f.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # F&C iteration map(utils.unfreeze_net, F_d.values()) utils.unfreeze_net(C) if opt.fix_emb: for f_d in F_d.values(): utils.freeze_net(f_d.word_emb) for f_d in F_d.values(): f_d.zero_grad() C.zero_grad() for domain in opt.domains: batch = utils.endless_get_next_batch( train_loaders, train_iters, domain) batch = tuple(t.to(opt.device) for t in batch) emb_ids, input_ids, input_mask, segment_ids, label_ids = batch inputs = emb_ids targets = label_ids shared_feat = torch.zeros(len(targets), opt.shared_hidden_size).to(opt.device) domain_feat = F_d[domain](inputs) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], None, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], None, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) if avg_acc > best_avg_acc: log.info('New best average validation accuracy: {}'.format(avg_acc)) best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) # for d in opt.domains: # if d in F_d: # torch.save(F_d[d].state_dict(), # '{}/net_F_d_{}.pth'.format(opt.model_save_file, d)) # torch.save(C.state_dict(), # '{}/netC.pth'.format(opt.model_save_file)) # end of training log.info('Best average validation accuracy: {}'.format(best_avg_acc)) return best_acc
best_dev_f = -10 bad_counter = 0 for epoch in range(opt.max_epoch): model.train() optimizer.zero_grad() # sum_cost = 0.0 correct, total = 0, 0 epoch_start = time.time() for i in range(num_iter): mention_inputs, _, sentences, mask, targets = utils.endless_get_next_batch(train_loader, train_iter) batch_output = model(mention_inputs, sentences, mask) cost = criterion(batch_output, targets) # sum_cost += cost.item() # train accuracy total += targets.size(0) _, pred = torch.max(batch_output, 1) # correct += (pred == targets).sum().sample_data[0] correct += (pred == targets).sum().item() cost.backward() optimizer.step() optimizer.zero_grad()
def train_shared(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders, F_s): C = None C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) F_s, C = F_s.to(opt.device), C.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain( *map(list, [F_s.parameters() if F_s else [], C.parameters()] + [])), lr=opt.learning_rate) # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # F&C iteration utils.unfreeze_net(F_s) utils.unfreeze_net(C) if opt.fix_emb: utils.freeze_net(F_s.word_emb) F_s.zero_grad() C.zero_grad() for domain in opt.domains: batch = utils.endless_get_next_batch( train_loaders, train_iters, domain) batch = tuple(t.to(opt.device) for t in batch) emb_ids, input_ids, input_mask, segment_ids, label_ids = batch inputs = emb_ids targets = label_ids _, shared_feat = F_s(input_ids, input_mask, segment_ids) domain_feat = torch.zeros(len(targets), opt.domain_hidden_size).to(opt.device) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.dev_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average validation accuracy: {}'.format(avg_acc)) log.info('Evaluating test sets:') test_acc = {} for domain in opt.dev_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info('Average test accuracy: {}'.format(avg_test_acc)) if avg_acc > best_avg_acc: log.info('New best average validation accuracy: {}'.format(avg_acc)) best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) # torch.save(C.state_dict(), # '{}/netC.pth'.format(opt.model_save_file)) # end of training log.info('Best average validation accuracy: {}'.format(best_avg_acc)) return best_acc
def train(train_sets, dev_sets, test_sets, unlabeled_sets, fold): """ train_sets, dev_sets, test_sets: dict[domain] -> TensorDataset For unlabeled domains, no train_sets are available """ # dataset loaders train_loaders, unlabeled_loaders = {}, {} train_iters, unlabeled_iters = {}, {} dev_loaders, test_loaders = {}, {} for domain in opt.domains: train_loaders[domain] = DataLoader(train_sets[domain], opt.batch_size, shuffle=True) train_iters[domain] = iter(train_loaders[domain]) for domain in opt.all_domains: dev_loaders[domain] = DataLoader(dev_sets[domain], opt.batch_size, shuffle=False) test_loaders[domain] = DataLoader(test_sets[domain], opt.batch_size, shuffle=False) if domain in opt.unlabeled_domains: uset = unlabeled_sets[domain] else: # for labeled domains, consider which data to use as unlabeled set if opt.unlabeled_data == 'both': uset = ConcatDataset( [train_sets[domain], unlabeled_sets[domain]]) elif opt.unlabeled_data == 'unlabeled': uset = unlabeled_sets[domain] elif opt.unlabeled_data == 'train': uset = train_sets[domain] else: raise Exception( f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}' ) unlabeled_loaders[domain] = DataLoader(uset, opt.batch_size, shuffle=True) unlabeled_iters[domain] = iter(unlabeled_loaders[domain]) # models F_s = None F_d = {} C, D = None, None if opt.model.lower() == 'mlp': F_s = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.shared_hidden_size, opt.dropout, opt.F_bn) for domain in opt.domains: F_d[domain] = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes, opt.domain_hidden_size, opt.dropout, opt.F_bn) else: raise Exception(f'Unknown model architecture {opt.model}') C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size, opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels, opt.dropout, opt.C_bn) D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size, len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn) F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device) for f_d in F_d.values(): f_d = f_d.to(opt.device) # optimizers optimizer = optim.Adam(itertools.chain( *map(list, [F_s.parameters() if F_s else [], C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate) optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate) # testing if opt.test_only: log.info(f'Loading model from {opt.model_save_file}...') F_s.load_state_dict( torch.load( os.path.join(opt.model_save_file, f'netF_s_fold{fold}.pth'))) for domain in opt.all_domains: if domain in F_d: F_d[domain].load_state_dict( torch.load( os.path.join(opt.model_save_file, f'net_F_d_{domain}_fold{fold}.pth'))) C.load_state_dict( torch.load( os.path.join(opt.model_save_file, f'netC_fold{fold}.pth'))) D.load_state_dict( torch.load( os.path.join(opt.model_save_file, f'netD_fold{fold}.pth'))) log.info('Evaluating validation sets:') acc = {} for domain in opt.all_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average validation accuracy: {avg_acc}') log.info('Evaluating test sets:') test_acc = {} for domain in opt.all_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average test accuracy: {avg_test_acc}') return {'valid': acc, 'test': test_acc} # training best_acc, best_avg_acc = defaultdict(float), 0.0 for epoch in range(opt.max_epoch): F_s.train() C.train() D.train() for f in F_d.values(): f.train() # training accuracy correct, total = defaultdict(int), defaultdict(int) # D accuracy d_correct, d_total = 0, 0 # conceptually view 1 epoch as 1 epoch of the first domain num_iter = len(train_loaders[opt.domains[0]]) for i in tqdm(range(num_iter)): # D iterations utils.freeze_net(F_s) map(utils.freeze_net, F_d.values()) utils.freeze_net(C) utils.unfreeze_net(D) # optional WGAN n_critic trick n_critic = opt.n_critic if opt.wgan_trick: if opt.n_critic > 0 and ((epoch == 0 and i < 25) or i % 500 == 0): n_critic = 100 for _ in range(n_critic): D.zero_grad() loss_d = {} # train on both labeled and unlabeled domains for domain in opt.all_domains: # targets not used d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs)) shared_feat = F_s(d_inputs) d_outputs = D(shared_feat) # D accuracy _, pred = torch.max(d_outputs, 1) d_total += len(d_inputs) if opt.loss.lower() == 'l2': _, tgt_indices = torch.max(d_targets, 1) d_correct += (pred == tgt_indices).sum().item() l_d = functional.mse_loss(d_outputs, d_targets) l_d.backward() else: d_correct += (pred == d_targets).sum().item() l_d = functional.nll_loss(d_outputs, d_targets) l_d.backward() loss_d[domain] = l_d.item() optimizerD.step() # F&C iteration utils.unfreeze_net(F_s) map(utils.unfreeze_net, F_d.values()) utils.unfreeze_net(C) utils.freeze_net(D) F_s.zero_grad() for f_d in F_d.values(): f_d.zero_grad() C.zero_grad() shared_feats, domain_feats = [], [] for domain in opt.domains: inputs, targets = utils.endless_get_next_batch( train_loaders, train_iters, domain) targets = targets.to(opt.device) shared_feat = F_s(inputs) shared_feats.append(shared_feat) domain_feat = F_d[domain](inputs) domain_feats.append(domain_feat) features = torch.cat((shared_feat, domain_feat), dim=1) c_outputs = C(features) l_c = functional.nll_loss(c_outputs, targets) l_c.backward(retain_graph=True) # training accuracy _, pred = torch.max(c_outputs, 1) total[domain] += targets.size(0) correct[domain] += (pred == targets).sum().item() # update F with D gradients on all domains for domain in opt.all_domains: d_inputs, _ = utils.endless_get_next_batch( unlabeled_loaders, unlabeled_iters, domain) shared_feat = F_s(d_inputs) d_outputs = D(shared_feat) if opt.loss.lower() == 'gr': d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs)) l_d = functional.nll_loss(d_outputs, d_targets) log.debug(f'D loss: {l_d.item()}') if opt.lambd > 0: l_d *= -opt.lambd elif opt.loss.lower() == 'bs': d_targets = utils.get_random_domain_label( opt.loss, len(d_inputs)) l_d = functional.kl_div(d_outputs, d_targets, size_average=False) if opt.lambd > 0: l_d *= opt.lambd elif opt.loss.lower() == 'l2': d_targets = utils.get_random_domain_label( opt.loss, len(d_inputs)) l_d = functional.mse_loss(d_outputs, d_targets) if opt.lambd > 0: l_d *= opt.lambd l_d.backward() if opt.model.lower() != 'lstm': log.debug( f'F_s norm: {F_s.net[-2].weight.grad.data.norm()}') optimizer.step() # end of epoch log.info('Ending epoch {}'.format(epoch + 1)) if d_total > 0: log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total)) log.info('Training accuracy:') log.info('\t'.join(opt.domains)) log.info('\t'.join( [str(100.0 * correct[d] / total[d]) for d in opt.domains])) log.info('Evaluating validation sets:') acc = {} for domain in opt.all_domains: acc[domain] = evaluate(domain, dev_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average validation accuracy: {avg_acc}') log.info('Evaluating test sets:') test_acc = {} for domain in opt.all_domains: test_acc[domain] = evaluate(domain, test_loaders[domain], F_s, F_d[domain] if domain in F_d else None, C) avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains) log.info(f'Average test accuracy: {avg_test_acc}') if avg_acc > best_avg_acc: log.info(f'New best average validation accuracy: {avg_acc}') best_acc['valid'] = acc best_acc['test'] = test_acc best_avg_acc = avg_acc with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf: pickle.dump(opt, ouf) torch.save( F_s.state_dict(), '{}/netF_s_fold{}.pth'.format(opt.model_save_file, fold)) for d in opt.domains: if d in F_d: torch.save( F_d[d].state_dict(), '{}/net_F_d_{}_fold{}.pth'.format( opt.model_save_file, d, fold)) torch.save(C.state_dict(), '{}/netC_fold{}.pth'.format(opt.model_save_file, fold)) torch.save(D.state_dict(), '{}/netD_fold{}.pth'.format(opt.model_save_file, fold)) # end of training log.info(f'Best average validation accuracy: {best_avg_acc}') return best_acc