def evaluate(data_loader, model, label_to_ix):
ix_to_label = {v: k for k, v in label_to_ix.items()}
correct = 0
total = 0
model.eval()
loader_it = iter(data_loader)
num_it = len(data_loader)
instances = []
for j in range(num_it):
mention_inputs, features, sentences, char_inputs, targets = utils.endless_get_next_batch(
data_loader, loader_it)
targets = utils.get_var(targets)
pred = model(mention_inputs, char_inputs)
_, y_pred = torch.max(pred, 1)
total += targets.size(0)
# correct += (y_pred == targets).sum().sample_data[0]
correct += (y_pred == targets).sum().item()
# output evaluate
pred_numpy = (y_pred.data).cpu().numpy()
y_pred_labels = [ix_to_label[ix] for ix in pred_numpy]
assert len(y_pred_labels) == len(
features), 'y_pred_labels and features have different lengths'
for i, pred_label in enumerate(y_pred_labels):
features[i][5] = pred_label
instances.append(features[i])
acc = 100.0 * correct / total
return acc, instances
def get_visual_features(num_iter, test_loaders, test_iters, F_d):
visual_features, senti_labels = None, None
with torch.no_grad():
for _ in tqdm(range(num_iter)):
for domain in opt.domains:
d_inputs, targets = utils.endless_get_next_batch(
test_loaders, test_iters, domain)
private_features = F_d[domain](d_inputs)
if visual_features is None:
visual_features = private_features
senti_labels = targets
else:
visual_features = torch.cat(
[visual_features, private_features], 0)
senti_labels = torch.cat([senti_labels, targets], 0)
return visual_features, senti_labels
dict_num_iter = len(dict_loader)
#start training dictionary
logging.info("batch_size: %s, dict_num_iter %s, train num_iter %s" %
(str(opt.batch_size), str(dict_num_iter), str(num_iter)))
for epoch in range(opt.dict_iteration):
epoch_start = time.time()
# sum_dict_cost = 0.0
correct_1, total_1 = 0, 0
model.train()
for i in range(dict_num_iter):
dict_inputs, _, dict_sentences, dict_char_inputs, dict_targets = utils.endless_get_next_batch(
dict_loader, dict_iter)
dict_targets = utils.get_var(dict_targets)
dict_batch_output = model(dict_inputs, dict_char_inputs)
dict_cost = criterion(dict_batch_output, dict_targets)
# sum_dict_cost += dict_cost.item()
# for dict training accuracy
total_1 += len(dict_inputs[1])
_, dict_pred = torch.max(dict_batch_output, 1)
correct_1 += (dict_pred == dict_targets).sum().item()
dict_cost.backward()
optimizer.step()
model.zero_grad()
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets):
"""
train_sets, dev_sets, test_sets: dict[domain] -> AmazonDataset
For unlabeled domains, no train_sets are available
"""
# dataset loaders
train_loaders, unlabeled_loaders = {}, {}
train_iters, unlabeled_iters = {}, {}
dev_loaders, test_loaders = {}, {}
my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size, shuffle=True, collate_fn=my_collate)
train_iters[domain] = iter(train_loaders[domain])
for domain in opt.dev_domains:
dev_loaders[domain] = DataLoader(dev_sets[domain],
opt.batch_size, shuffle=False, collate_fn=my_collate)
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size, shuffle=False, collate_fn=my_collate)
for domain in opt.all_domains:
if domain in opt.unlabeled_domains:
uset = unlabeled_sets[domain]
else:
# for labeled domains, consider which data to use as unlabeled set
if opt.unlabeled_data == 'both':
uset = ConcatDataset([train_sets[domain], unlabeled_sets[domain]])
elif opt.unlabeled_data == 'unlabeled':
uset = unlabeled_sets[domain]
elif opt.unlabeled_data == 'train':
uset = train_sets[domain]
else:
raise Exception('Unknown options for the unlabeled data usage: {}'.format(opt.unlabeled_data))
unlabeled_loaders[domain] = DataLoader(uset,
opt.batch_size, shuffle=True, collate_fn=my_collate)
unlabeled_iters[domain] = iter(unlabeled_loaders[domain])
# models
F_s = None
C, D = None, None
if opt.model.lower() == 'dan':
F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
elif opt.model.lower() == 'lstm':
F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
elif opt.model.lower() == 'cnn':
F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.kernel_num, opt.kernel_sizes, opt.dropout)
else:
raise Exception('Unknown model architecture {}'.format(opt.model))
C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn)
F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(
*map(list, [F_s.parameters() if F_s else [], C.parameters()] + [])),
lr=opt.learning_rate)
optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate)
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
D.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# D iterations
utils.freeze_net(F_s)
utils.freeze_net(C)
utils.unfreeze_net(D)
# WGAN n_critic trick since D trains slower
n_critic = opt.n_critic
if opt.wgan_trick:
if opt.n_critic > 0 and ((epoch == 0 and i < 25) or i % 500 == 0):
n_critic = 100
for _ in range(n_critic):
D.zero_grad()
loss_d = {}
# train on both labeled and unlabeled domains
for domain in opt.all_domains:
# targets not used
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1]))
shared_feat = F_s(d_inputs)
d_outputs = D(shared_feat)
# D accuracy
_, pred = torch.max(d_outputs, 1)
d_total += len(d_inputs[1])
if opt.loss.lower() == 'l2':
_, tgt_indices = torch.max(d_targets, 1)
d_correct += (pred == tgt_indices).sum().item()
l_d = functional.mse_loss(d_outputs, d_targets)
l_d.backward()
else:
d_correct += (pred == d_targets).sum().item()
l_d = functional.nll_loss(d_outputs, d_targets)
l_d.backward()
loss_d[domain] = l_d.item()
optimizerD.step()
# F&C iteration
utils.unfreeze_net(F_s)
utils.unfreeze_net(C)
utils.freeze_net(D)
if opt.fix_emb:
utils.freeze_net(F_s.word_emb)
F_s.zero_grad()
C.zero_grad()
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
shared_feat = F_s(inputs)
domain_feat = torch.zeros(len(targets), opt.domain_hidden_size).to(opt.device)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
# update F with D gradients on all domains
for domain in opt.all_domains:
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
shared_feat = F_s(d_inputs)
d_outputs = D(shared_feat)
if opt.loss.lower() == 'gr':
d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs[1]))
l_d = functional.nll_loss(d_outputs, d_targets)
if opt.lambd > 0:
l_d *= -opt.lambd
elif opt.loss.lower() == 'bs':
d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs[1]))
l_d = functional.kl_div(d_outputs, d_targets, size_average=False)
if opt.lambd > 0:
l_d *= opt.lambd
elif opt.loss.lower() == 'l2':
d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs[1]))
l_d = functional.mse_loss(d_outputs, d_targets)
if opt.lambd > 0:
l_d *= opt.lambd
l_d.backward()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain],
F_s, None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain],
F_s, None, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
if avg_acc > best_avg_acc:
log.info('New best average validation accuracy: {}'.format(avg_acc))
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf:
pickle.dump(opt, ouf)
torch.save(F_s.state_dict(),
'{}/netF_s.pth'.format(opt.model_save_file))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.model_save_file))
torch.save(D.state_dict(),
'{}/netD.pth'.format(opt.model_save_file))
# end of training
log.info('Best average validation accuracy: {}'.format(best_avg_acc))
return best_acc
def train(vocabs, char_vocab, tag_vocab, train_sets, dev_sets, test_sets, unlabeled_sets):
"""
train_sets, dev_sets, test_sets: dict[lang] -> AmazonDataset
For unlabeled langs, no train_sets are available
"""
# dataset loaders
train_loaders, unlabeled_loaders = {}, {}
train_iters, unlabeled_iters, d_unlabeled_iters = {}, {}, {}
dev_loaders, test_loaders = {}, {}
my_collate = utils.sorted_collate if opt.model=='lstm' else utils.unsorted_collate
for lang in opt.langs:
train_loaders[lang] = DataLoader(train_sets[lang],
opt.batch_size, shuffle=True, collate_fn = my_collate)
train_iters[lang] = iter(train_loaders[lang])
for lang in opt.dev_langs:
dev_loaders[lang] = DataLoader(dev_sets[lang],
opt.batch_size, shuffle=False, collate_fn = my_collate)
test_loaders[lang] = DataLoader(test_sets[lang],
opt.batch_size, shuffle=False, collate_fn = my_collate)
for lang in opt.all_langs:
if lang in opt.unlabeled_langs:
uset = unlabeled_sets[lang]
else:
# for labeled langs, consider which data to use as unlabeled set
if opt.unlabeled_data == 'both':
uset = ConcatDataset([train_sets[lang], unlabeled_sets[lang]])
elif opt.unlabeled_data == 'unlabeled':
uset = unlabeled_sets[lang]
elif opt.unlabeled_data == 'train':
uset = train_sets[lang]
else:
raise Exception(f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}')
unlabeled_loaders[lang] = DataLoader(uset,
opt.batch_size, shuffle=True, collate_fn = my_collate)
unlabeled_iters[lang] = iter(unlabeled_loaders[lang])
d_unlabeled_iters[lang] = iter(unlabeled_loaders[lang])
# embeddings
emb = MultiLangWordEmb(vocabs, char_vocab, opt.use_wordemb, opt.use_charemb).to(opt.device)
# models
F_s = None
F_p = None
C, D = None, None
num_experts = len(opt.langs)+1 if opt.expert_sp else len(opt.langs)
if opt.model.lower() == 'lstm':
if opt.shared_hidden_size > 0:
F_s = LSTMFeatureExtractor(opt.total_emb_size, opt.F_layers, opt.shared_hidden_size,
opt.word_dropout, opt.dropout, opt.bdrnn)
if opt.private_hidden_size > 0:
if not opt.concat_sp:
assert opt.shared_hidden_size == opt.private_hidden_size, "shared dim != private dim when using add_sp!"
F_p = nn.Sequential(
LSTMFeatureExtractor(opt.total_emb_size, opt.F_layers, opt.private_hidden_size,
opt.word_dropout, opt.dropout, opt.bdrnn),
MixtureOfExperts(opt.MoE_layers, opt.private_hidden_size,
len(opt.langs), opt.private_hidden_size,
opt.private_hidden_size, opt.dropout, opt.MoE_bn, False)
)
else:
raise Exception(f'Unknown model architecture {opt.model}')
if opt.C_MoE:
C = SpMixtureOfExperts(opt.C_layers, opt.shared_hidden_size, opt.private_hidden_size, opt.concat_sp,
num_experts, opt.shared_hidden_size + opt.private_hidden_size, len(tag_vocab),
opt.mlp_dropout, opt.C_bn)
else:
C = SpMlpTagger(opt.C_layers, opt.shared_hidden_size, opt.private_hidden_size, opt.concat_sp,
opt.shared_hidden_size + opt.private_hidden_size, len(tag_vocab),
opt.mlp_dropout, opt.C_bn)
if opt.shared_hidden_size > 0 and opt.n_critic > 0:
if opt.D_model.lower() == 'lstm':
d_args = {
'num_layers': opt.D_lstm_layers,
'input_size': opt.shared_hidden_size,
'hidden_size': opt.shared_hidden_size,
'word_dropout': opt.D_word_dropout,
'dropout': opt.D_dropout,
'bdrnn': opt.D_bdrnn,
'attn_type': opt.D_attn
}
elif opt.D_model.lower() == 'cnn':
d_args = {
'num_layers': 1,
'input_size': opt.shared_hidden_size,
'hidden_size': opt.shared_hidden_size,
'kernel_num': opt.D_kernel_num,
'kernel_sizes': opt.D_kernel_sizes,
'word_dropout': opt.D_word_dropout,
'dropout': opt.D_dropout
}
else:
d_args = None
if opt.D_model.lower() == 'mlp':
D = MLPLanguageDiscriminator(opt.D_layers, opt.shared_hidden_size,
opt.shared_hidden_size, len(opt.all_langs), opt.loss, opt.D_dropout, opt.D_bn)
else:
D = LanguageDiscriminator(opt.D_model, opt.D_layers,
opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_langs), opt.D_dropout, opt.D_bn, d_args)
if opt.use_data_parallel:
F_s, C, D = nn.DataParallel(F_s).to(opt.device) if F_s else None, nn.DataParallel(C).to(opt.device), nn.DataParallel(D).to(opt.device) if D else None
else:
F_s, C, D = F_s.to(opt.device) if F_s else None, C.to(opt.device), D.to(opt.device) if D else None
if F_p:
if opt.use_data_parallel:
F_p = nn.DataParallel(F_p).to(opt.device)
else:
F_p = F_p.to(opt.device)
# optimizers
optimizer = optim.Adam(filter(lambda p: p.requires_grad, itertools.chain(*map(list,
[emb.parameters(), F_s.parameters() if F_s else [], \
C.parameters(), F_p.parameters() if F_p else []]))),
lr=opt.learning_rate,
weight_decay=opt.weight_decay)
if D:
optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate, weight_decay=opt.D_weight_decay)
# testing
if opt.test_only:
log.info(f'Loading model from {opt.model_save_file}...')
if F_s:
F_s.load_state_dict(torch.load(os.path.join(opt.model_save_file,
f'netF_s.pth')))
for lang in opt.all_langs:
F_p.load_state_dict(torch.load(os.path.join(opt.model_save_file,
f'net_F_p.pth')))
C.load_state_dict(torch.load(os.path.join(opt.model_save_file,
f'netC.pth')))
if D:
D.load_state_dict(torch.load(os.path.join(opt.model_save_file,
f'netD.pth')))
log.info('Evaluating validation sets:')
acc = {}
log.info(dev_loaders)
log.info(vocabs)
for lang in opt.all_langs:
acc[lang] = evaluate(f'{lang}_dev', dev_loaders[lang], vocabs[lang], tag_vocab,
emb, lang, F_s, F_p, C)
avg_acc = sum([acc[d] for d in opt.dev_langs]) / len(opt.dev_langs)
log.info(f'Average validation accuracy: {avg_acc}')
log.info('Evaluating test sets:')
test_acc = {}
for lang in opt.all_langs:
test_acc[lang] = evaluate(f'{lang}_test', test_loaders[lang], vocabs[lang], tag_vocab,
emb, lang, F_s, F_p, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_langs]) / len(opt.dev_langs)
log.info(f'Average test accuracy: {avg_test_acc}')
return {'valid': acc, 'test': test_acc}
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
epochs_since_decay = 0
# lambda scheduling
if opt.lambd > 0 and opt.lambd_schedule:
opt.lambd_orig = opt.lambd
num_iter = int(utils.gmean([len(train_loaders[l]) for l in opt.langs]))
# adapt max_epoch
if opt.max_epoch > 0 and num_iter * opt.max_epoch < 15000:
opt.max_epoch = 15000 // num_iter
log.info(f"Setting max_epoch to {opt.max_epoch}")
for epoch in range(opt.max_epoch):
emb.train()
if F_s:
F_s.train()
C.train()
if D:
D.train()
if F_p:
F_p.train()
# lambda scheduling
if hasattr(opt, 'lambd_orig') and opt.lambd_schedule:
if epoch == 0:
opt.lambd = opt.lambd_orig
elif epoch == 5:
opt.lambd = 10 * opt.lambd_orig
elif epoch == 15:
opt.lambd = 100 * opt.lambd_orig
log.info(f'Scheduling lambda = {opt.lambd}')
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
gate_correct = defaultdict(int)
c_gate_correct = defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
for i in tqdm(range(num_iter), ascii=True):
# D iterations
if opt.shared_hidden_size > 0:
utils.freeze_net(emb)
utils.freeze_net(F_s)
utils.freeze_net(F_p)
utils.freeze_net(C)
utils.unfreeze_net(D)
# WGAN n_critic trick since D trains slower
n_critic = opt.n_critic
if opt.wgan_trick:
if opt.n_critic>0 and ((epoch==0 and i<25) or i%500==0):
n_critic = 100
for _ in range(n_critic):
D.zero_grad()
loss_d = {}
lang_features = {}
# train on both labeled and unlabeled langs
for lang in opt.all_langs:
# targets not used
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, d_unlabeled_iters, lang)
d_inputs, d_lengths, mask, d_chars, d_char_lengths = d_inputs
d_embeds = emb(lang, d_inputs, d_chars, d_char_lengths)
shared_feat = F_s((d_embeds, d_lengths))
if opt.grad_penalty != 'none':
lang_features[lang] = shared_feat.detach()
if opt.D_model.lower() == 'mlp':
d_outputs = D(shared_feat)
# if token-level D, we can reuse the gate label generator
d_targets = utils.get_gate_label(d_outputs, lang, mask, False, all_langs=True)
d_total += torch.sum(d_lengths).item()
else:
d_outputs = D((shared_feat, d_lengths))
d_targets = utils.get_lang_label(opt.loss, lang, len(d_lengths))
d_total += len(d_lengths)
# D accuracy
_, pred = torch.max(d_outputs, -1)
# d_total += len(d_lengths)
d_correct += (pred==d_targets).sum().item()
if opt.use_data_parallel:
l_d = functional.nll_loss(d_outputs.view(-1, D.module.num_langs),
d_targets.view(-1), ignore_index=-1)
else:
l_d = functional.nll_loss(d_outputs.view(-1, D.num_langs),
d_targets.view(-1), ignore_index=-1)
l_d.backward()
loss_d[lang] = l_d.item()
# gradient penalty
if opt.grad_penalty != 'none':
gp = utils.calc_gradient_penalty(D, lang_features,
onesided=opt.onesided_gp, interpolate=(opt.grad_penalty=='wgan'))
gp.backward()
optimizerD.step()
# F&C iteration
utils.unfreeze_net(emb)
if opt.use_wordemb and opt.fix_emb:
for lang in emb.langs:
emb.wordembs[lang].weight.requires_grad = False
if opt.use_charemb and opt.fix_charemb:
emb.charemb.weight.requires_grad = False
utils.unfreeze_net(F_s)
utils.unfreeze_net(F_p)
utils.unfreeze_net(C)
utils.freeze_net(D)
emb.zero_grad()
if F_s:
F_s.zero_grad()
if F_p:
F_p.zero_grad()
C.zero_grad()
# optimizer.zero_grad()
for lang in opt.langs:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, lang)
inputs, lengths, mask, chars, char_lengths = inputs
bs, seq_len = inputs.size()
embeds = emb(lang, inputs, chars, char_lengths)
shared_feat, private_feat = None, None
if opt.shared_hidden_size > 0:
shared_feat = F_s((embeds, lengths))
if opt.private_hidden_size > 0:
private_feat, gate_outputs = F_p((embeds, lengths))
if opt.C_MoE:
c_outputs, c_gate_outputs = C((shared_feat, private_feat))
else:
c_outputs = C((shared_feat, private_feat))
# targets are padded with -1
l_c = functional.nll_loss(c_outputs.view(bs*seq_len, -1),
targets.view(-1), ignore_index=-1)
# gate loss
if F_p:
gate_targets = utils.get_gate_label(gate_outputs, lang, mask, False)
l_gate = functional.cross_entropy(gate_outputs.view(bs*seq_len, -1),
gate_targets.view(-1), ignore_index=-1)
l_c += opt.gate_loss_weight * l_gate
_, gate_pred = torch.max(gate_outputs.view(bs*seq_len, -1), -1)
gate_correct[lang] += (gate_pred == gate_targets.view(-1)).sum().item()
if opt.C_MoE and opt.C_gate_loss_weight > 0:
c_gate_targets = utils.get_gate_label(c_gate_outputs, lang, mask, opt.expert_sp)
_, c_gate_pred = torch.max(c_gate_outputs.view(bs*seq_len, -1), -1)
if opt.expert_sp:
l_c_gate = functional.binary_cross_entropy_with_logits(
mask.unsqueeze(-1) * c_gate_outputs, c_gate_targets)
c_gate_correct[lang] += torch.index_select(c_gate_targets.view(bs*seq_len, -1),
-1, c_gate_pred.view(bs*seq_len)).sum().item()
else:
l_c_gate = functional.cross_entropy(c_gate_outputs.view(bs*seq_len, -1),
c_gate_targets.view(-1), ignore_index=-1)
c_gate_correct[lang] += (c_gate_pred == c_gate_targets.view(-1)).sum().item()
l_c += opt.C_gate_loss_weight * l_c_gate
l_c.backward()
_, pred = torch.max(c_outputs, -1)
total[lang] += torch.sum(lengths).item()
correct[lang] += (pred == targets).sum().item()
# update F with D gradients on all langs
if D:
for lang in opt.all_langs:
inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, lang)
inputs, lengths, mask, chars, char_lengths = inputs
embeds = emb(lang, inputs, chars, char_lengths)
shared_feat = F_s((embeds, lengths))
# d_outputs = D((shared_feat, lengths))
if opt.D_model.lower() == 'mlp':
d_outputs = D(shared_feat)
# if token-level D, we can reuse the gate label generator
d_targets = utils.get_gate_label(d_outputs, lang, mask, False, all_langs=True)
else:
d_outputs = D((shared_feat, lengths))
d_targets = utils.get_lang_label(opt.loss, lang, len(lengths))
if opt.use_data_parallel:
l_d = functional.nll_loss(d_outputs.view(-1, D.module.num_langs),
d_targets.view(-1), ignore_index=-1)
else:
l_d = functional.nll_loss(d_outputs.view(-1, D.num_langs),
d_targets.view(-1), ignore_index=-1)
if opt.lambd > 0:
l_d *= -opt.lambd
l_d.backward()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch+1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0*d_correct/d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.langs))
log.info('\t'.join([str(100.0*correct[d]/total[d]) for d in opt.langs]))
log.info('Gate accuracy:')
log.info('\t'.join([str(100.0*gate_correct[d]/total[d]) for d in opt.langs]))
log.info('Tagger Gate accuracy:')
log.info('\t'.join([str(100.0*c_gate_correct[d]/total[d]) for d in opt.langs]))
log.info('Evaluating validation sets:')
acc = {}
for lang in opt.dev_langs:
acc[lang] = evaluate(f'{lang}_dev', dev_loaders[lang], vocabs[lang], tag_vocab,
emb, lang, F_s, F_p, C)
avg_acc = sum([acc[d] for d in opt.dev_langs]) / len(opt.dev_langs)
log.info(f'Average validation accuracy: {avg_acc}')
log.info('Evaluating test sets:')
test_acc = {}
for lang in opt.dev_langs:
test_acc[lang] = evaluate(f'{lang}_test', test_loaders[lang], vocabs[lang], tag_vocab,
emb, lang, F_s, F_p, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_langs]) / len(opt.dev_langs)
log.info(f'Average test accuracy: {avg_test_acc}')
if avg_acc > best_avg_acc:
epochs_since_decay = 0
log.info(f'New best average validation accuracy: {avg_acc}')
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf:
pickle.dump(opt, ouf)
if F_s:
torch.save(F_s.state_dict(),
'{}/netF_s.pth'.format(opt.model_save_file))
torch.save(emb.state_dict(),
'{}/net_emb.pth'.format(opt.model_save_file))
if F_p:
torch.save(F_p.state_dict(),
'{}/net_F_p.pth'.format(opt.model_save_file))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.model_save_file))
if D:
torch.save(D.state_dict(),
'{}/netD.pth'.format(opt.model_save_file))
else:
epochs_since_decay += 1
if opt.lr_decay < 1 and epochs_since_decay >= opt.lr_decay_epochs:
epochs_since_decay = 0
old_lr = optimizer.param_groups[0]['lr']
optimizer.param_groups[0]['lr'] = old_lr * opt.lr_decay
log.info(f'Decreasing LR to {old_lr * opt.lr_decay}')
# end of training
log.info(f'Best average validation accuracy: {best_avg_acc}')
return best_acc
def train(target_domain_idx, train_sets, test_sets):
"""
train_sets, test_sets: unlabeled domain(dev domain) -> AmazonDataset
"""
# test dataset
print(train_sets[0])
print(test_sets[0])
print(train_sets[0][0].shape)
print(test_sets[0][1].shape)
# ---------------------- dataloader ---------------------- #
train_loaders = DataLoader(train_sets, opt.batch_size, shuffle=False)
test_loaders = DataLoader(test_sets, opt.batch_size, shuffle=False)
# ---------------------- model initialization ---------------------- #
F_s = None
F_d = {}
C = None
if opt.model.lower() == 'mlp':
F_s = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes,
opt.shared_hidden_size, opt.dropout, opt.F_bn)
for domain in opt.domains:
F_d[domain] = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes,
opt.domain_hidden_size, opt.dropout, opt.F_bn)
else:
raise Exception(f'Unknown model architecture {opt.model}')
C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn)
# 转移到gpu上
F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
# ---------------------- load pre-training model ---------------------- #
log.info(f'Loading model from {opt.exp2_model_save_file}...')
F_s.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file,
f'netF_s.pth')))
for domain in opt.all_domains:
if domain in F_d:
F_d[domain].load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file,
f'net_F_d_{domain}.pth')))
C.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file,
f'netC.pth')))
D.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file,
f'netD.pth')))
# ---------------------- get fake label(hard label) ---------------------- #
log.info('Get fake label:')
pseudo_labels, _ = genarate_labels(target_domain_idx, False, train_loaders, F_s, F_d, C, D)
# ********************** Test the accuracy of the label prediction ********************** #
test_pseudo_labels, targets_total = genarate_labels(target_domain_idx, True, test_loaders, F_s, F_d, C, D)
label_correct_acc = calc_label_prediction(test_pseudo_labels, targets_total)
log.info(f'the correct rate of label prediction: {label_correct_acc}')
# ********************** Test the accuracy of the label prediction ********************** #
# ------------------------------- 构造target数据集 ------------------------------- #
target_dataset = Subset(train_sets, pseudo_labels)
target_dataloader_labelled = DataLoader(target_dataset, opt.batch_size, shuffle=True)
target_train_iters = iter(target_dataloader_labelled)
# ------------------------------- F_d_target模型以及一些必要的参数定义 ------------------------------- #
F_d_target = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes,
opt.domain_hidden_size, opt.dropout, opt.F_bn)
F_d_target = F_d_target.to(opt.device)
# ******************************* 加载预训练模型的权重，不用再从头训练 ******************************* #
f_d_choice = random.randint(0, len(opt.domains))
F_d_target.load_state_dict(torch.load(os.path.join(opt.exp2_model_save_file,
f'net_F_d_{opt.domains[f_d_choice]}.pth')))
optimizer_F_d_target = optim.Adam(F_d_target.parameters(), lr=opt.learning_rate)
# ------------------------------- 测试一下只利用shared feature的准确率 ------------------------------- #
log.info('Evaluating test sets only on shared feature:')
test_acc = evaluate(opt.dev_domains, test_loaders, F_s, None, C)
log.info(f'test accuracy: {test_acc}')
for epoch in range(opt.max_epoch):
F_d_target.train()
C.train()
# training accuracy
correct, total = 0, 0
num_iter = len(target_dataloader_labelled)
print(num_iter)
for _ in tqdm(range(num_iter)):
utils.freeze_net(F_s)
C.zero_grad()
F_d_target.zero_grad()
inputs, targets = utils.endless_get_next_batch(target_dataloader_labelled, target_train_iters)
targets = targets.to(opt.device)
shared_feat = F_s(inputs)
domain_feat = F_d_target(inputs)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward()
_, pred_idx = torch.max(c_outputs, 1)
total += targets.size(0)
correct += (pred_idx == targets).sum().item()
optimizer_F_d_target.step()
# 暂时留下一个问题，这里没有写验证集，因为如果这里要设置验证集的话，需要将训练集切割
# 因为unlabeled domain里面只有2000个label sample以及raw_unlabeled_sets(四千多张)
# 后者训练时用，前者测试的时候用
# end of epoch
print("correct is %d" % correct)
print("total is %d" % total)
log.info('Ending epoch {}'.format(epoch + 1))
log.info('Training accuracy:')
log.info(opt.unlabeled_domains)
log.info(str(100.0 * correct / total))
# 训练过程中的验证集
print("correct is %d" % correct)
print("total is %d" % total)
log.info('Ending epoch {}'.format(epoch + 1))
log.info('Training accuracy:')
log.info('\t'.join(opt.unlabeled_domains))
log.info(str(100.0 * correct / total))
# 保存模型
torch.save(F_d_target.state_dict(),
'{}/net_F_d_target.pth'.format(opt.exp2_target_model_save_file))
torch.save(C.state_dict(),
'{}/netC_target.pth'.format(opt.exp2_target_model_save_file))
# 在测试集上测试，选择2000张有label的标签数据
# 方便对比与单独训练有label时的情况
log.info('Evaluating test sets:')
test_acc = evaluate(opt.dev_domains, test_loaders, F_s, F_d_target, C)
log.info(f'test accuracy: {test_acc}')
return test_acc
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets):
"""
train_sets, dev_sets, test_sets: dict[domain] -> AmazonDataset
For unlabeled domains, no train_sets are available
"""
# dataset loaders
train_loaders, unlabeled_loaders = {}, {}
train_iters, unlabeled_iters = {}, {}
dev_loaders, test_loaders = {}, {}
my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size,
shuffle=True,
collate_fn=my_collate)
train_iters[domain] = iter(train_loaders[domain])
for domain in opt.dev_domains:
dev_loaders[domain] = DataLoader(dev_sets[domain],
opt.batch_size,
shuffle=False,
collate_fn=my_collate)
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size,
shuffle=False,
collate_fn=my_collate)
for domain in opt.all_domains:
if domain in opt.unlabeled_domains:
uset = unlabeled_sets[domain]
else:
# for labeled domains, consider which data to use as unlabeled set
if opt.unlabeled_data == 'both':
uset = ConcatDataset(
[train_sets[domain], unlabeled_sets[domain]])
elif opt.unlabeled_data == 'unlabeled':
uset = unlabeled_sets[domain]
elif opt.unlabeled_data == 'train':
uset = train_sets[domain]
else:
raise Exception(
f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}'
)
unlabeled_loaders[domain] = DataLoader(uset,
opt.batch_size,
shuffle=True,
collate_fn=my_collate)
unlabeled_iters[domain] = iter(unlabeled_loaders[domain])
# model
F_s = None
F_d = {}
C, D = None, None
if opt.model.lower() == 'dan':
F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
for domain in opt.domains:
F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers,
opt.domain_hidden_size,
opt.sum_pooling, opt.dropout,
opt.F_bn)
elif opt.model.lower() == 'lstm':
F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
for domain in opt.domains:
F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers,
opt.domain_hidden_size,
opt.dropout, opt.bdrnn,
opt.attn)
elif opt.model.lower() == 'cnn':
F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.kernel_num, opt.kernel_sizes,
opt.dropout)
for domain in opt.domains:
F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers,
opt.domain_hidden_size,
opt.kernel_num, opt.kernel_sizes,
opt.dropout)
else:
raise Exception(f'Unknown model architecture {opt.model}')
C = SentimentClassifier(opt.C_layers,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.num_labels, opt.dropout, opt.C_bn)
D = DomainClassifier(opt.D_layers,
opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn)
F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(
*map(list, [F_s.parameters() if F_s else [],
C.parameters()] + [f.parameters() for f in F_d.values()])),
lr=opt.learning_rate)
optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate)
# testing
if opt.test_only:
log.info(f'Loading model from {opt.exp3_model_save_file}...')
if F_s:
F_s.load_state_dict(
torch.load(
os.path.join(opt.exp3_model_save_file, f'netF_s.pth')))
for domain in opt.all_domains:
if domain in F_d:
F_d[domain].load_state_dict(
torch.load(
os.path.join(opt.exp3_model_save_file,
f'net_F_d_{domain}.pth')))
C.load_state_dict(
torch.load(os.path.join(opt.exp3_model_save_file, f'netC.pth')))
D.load_state_dict(
torch.load(os.path.join(opt.exp3_model_save_file, f'netD.pth')))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.all_domains:
acc[domain] = evaluate(domain, dev_loaders[domain], F_s,
F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average validation accuracy: {avg_acc}')
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.all_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_s,
F_d[domain] if domain in F_d else None,
C)
avg_test_acc = sum([test_acc[d]
for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average test accuracy: {avg_test_acc}')
return {'valid': acc, 'test': test_acc}
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
D.train()
LAMBDA = 3
lambda1 = 0.1
lambda2 = 0.1
for f in F_d.values():
f.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
shared_d_correct, private_d_correct, d_total = 0, 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# D iterations
utils.freeze_net(F_s)
map(utils.freeze_net, F_d.values())
utils.freeze_net(C)
utils.unfreeze_net(D)
# WGAN n_critic trick since D trains slower
# ********************** D iterations on all domains ********************** #
# ---------------------- update D with D gradients on all domains ---------------------- #
n_critic = opt.n_critic
if opt.wgan_trick:
if opt.n_critic > 0 and ((epoch == 0 and i < 25)
or i % 500 == 0):
n_critic = 100
for _ in range(n_critic):
D.zero_grad()
loss_d = {}
# train on both labeled and unlabeled domains
for domain in opt.all_domains:
# targets not usedndless_get_next_batch(
# unlabeled_loaders, unlabeled_iters, domain)
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
d_targets = utils.get_domain_label(opt.loss, domain,
len(d_inputs[1]))
shared_feat = F_s(d_inputs)
shared_d_outputs = D(shared_feat)
_, shared_pred = torch.max(shared_d_outputs, 1)
if domain != opt.dev_domains[0]:
private_feat = F_d[domain](d_inputs)
private_d_outputs = D(private_feat)
_, private_pred = torch.max(private_d_outputs, 1)
d_total += len(d_inputs[1])
if opt.loss.lower() == 'l2':
_, tgt_indices = torch.max(d_targets, 1)
shared_d_correct += (
shared_pred == tgt_indices).sum().item()
shared_l_d = functional.mse_loss(
shared_d_outputs, d_targets)
private_l_d = 0.0
if domain != opt.dev_domains[0]:
private_d_correct += (
private_pred == tgt_indices).sum().item()
private_l_d = functional.mse_loss(
private_d_outputs, d_targets) / len(
opt.domains)
l_d_sum = shared_l_d + private_l_d
l_d_sum.backward()
else:
shared_d_correct += (
shared_pred == d_targets).sum().item()
shared_l_d = functional.nll_loss(
shared_d_outputs, d_targets)
private_l_d = 0.0
if domain != opt.dev_domains[0]:
private_d_correct += (
private_pred == d_targets).sum().item()
private_l_d = functional.nll_loss(
private_d_outputs, d_targets) / len(
opt.domains)
l_d_sum = shared_l_d + private_l_d
l_d_sum.backward()
loss_d[domain] = l_d_sum.item()
optimizerD.step()
# ---------------------- update D with C gradients on all domains ---------------------- #
# ********************** D iterations on all domains ********************** #
# ********************** F&C iteration ********************** #
# ---------------------- update F_s & F_ds with C gradients on all labeled domains ---------------------- #
# F&C iteration
utils.unfreeze_net(F_s)
map(utils.unfreeze_net, F_d.values())
utils.unfreeze_net(C)
utils.freeze_net(D)
if opt.fix_emb:
utils.freeze_net(F_s.word_emb)
for f_d in F_d.values():
utils.freeze_net(f_d.word_emb)
F_s.zero_grad()
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
shared_feat = F_s(inputs)
domain_feat = F_d[domain](inputs)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
loss_part_1 = functional.nll_loss(c_outputs, targets)
targets = targets.unsqueeze(1)
targets_onehot = torch.FloatTensor(opt.batch_size, 2)
targets_onehot.zero_()
targets_onehot.scatter_(1, targets.cpu(), 1)
targets_onehot = targets_onehot.to(opt.device)
loss_part_2 = lambda1 * margin_regularization(
inputs, targets_onehot, F_d[domain], LAMBDA)
loss_part_3 = -lambda2 * center_point_constraint(
domain_feat, targets)
print("lambda1: " + str(lambda1))
print("lambda2: " + str(lambda2))
print("loss_part_1: " + str(loss_part_1))
print("loss_part_2: " + str(loss_part_2))
print("loss_part_3: " + str(loss_part_3))
l_c = loss_part_1 + loss_part_2 + loss_part_3
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
# ---------------------- update F_s & F_ds with C gradients on all labeled domains ---------------------- #
# ---------------------- update F_s with D gradients on all domains ---------------------- #
# update F with D gradients on all domains
for domain in opt.all_domains:
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
shared_feat = F_s(d_inputs)
shared_d_outputs = D(shared_feat)
if domain != opt.dev_domains[0]:
private_feat = F_d[domain](d_inputs)
private_d_outputs = D(private_feat)
l_d_sum = None
if opt.loss.lower() == 'gr':
d_targets = utils.get_domain_label(opt.loss, domain,
len(d_inputs[1]))
shared_l_d = functional.nll_loss(shared_d_outputs,
d_targets)
private_l_d, l_d_sum = 0.0, 0.0
if domain != opt.dev_domains[0]:
# 注意这边的loss function
private_l_d = functional.nll_loss(
private_d_outputs, d_targets) * -1. / len(
opt.domains)
if opt.shared_lambd > 0:
l_d_sum = shared_l_d * opt.shared_lambd * -1.
else:
l_d_sum = shared_l_d * -1.
if opt.private_lambd > 0:
l_d_sum += private_l_d * opt.private_lambd * -1.
else:
l_d_sum += private_l_d * -1.
elif opt.loss.lower() == 'bs':
d_targets = utils.get_random_domain_label(
opt.loss, len(d_inputs[1]))
shared_l_d = functional.kl_div(shared_d_outputs,
d_targets,
size_average=False)
private_l_d, l_d_sum = 0.0, 0.0
if domain != opt.dev_domains[0]:
private_l_d = functional.kl_div(private_d_outputs, d_targets, size_average=False) \
* -1. / len(opt.domains)
if opt.shared_lambd > 0:
l_d_sum = shared_l_d * opt.shared_lambd
else:
l_d_sum = shared_l_d
if opt.private_lambd > 0:
l_d_sum += private_l_d * opt.private_lambd
else:
l_d_sum += private_l_d
elif opt.loss.lower() == 'l2':
d_targets = utils.get_random_domain_label(
opt.loss, len(d_inputs[1]))
shared_l_d = functional.mse_loss(shared_d_outputs,
d_targets)
private_l_d, l_d_sum = 0.0, 0.0
if domain != opt.dev_domains[0]:
private_l_d = functional.mse_loss(
private_d_outputs, d_targets) * -1. / len(
opt.domains)
if opt.shared_lambd > 0:
l_d_sum = shared_l_d * opt.shared_lambd
else:
l_d_sum = shared_l_d
if opt.private_lambd > 0:
l_d_sum += private_l_d * opt.private_lambd
else:
l_d_sum += private_l_d
l_d_sum.backward()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('shared D Training Accuracy: {}%'.format(
100.0 * shared_d_correct / d_total))
log.info('private D Training Accuracy(average): {}%'.format(
100.0 * private_d_correct / len(opt.domains) / d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join(
[str(100.0 * correct[d] / total[d]) for d in opt.domains]))
# 验证集上验证实验
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain], F_s,
F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average validation accuracy: {avg_acc}')
# 测试集上验证实验
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_s,
F_d[domain] if domain in F_d else None,
C)
avg_test_acc = sum([test_acc[d]
for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average test accuracy: {avg_test_acc}')
# 保存模型
if avg_acc > best_avg_acc:
log.info(f'New best average validation accuracy: {avg_acc}')
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.exp3_model_save_file, 'options.pkl'),
'wb') as ouf:
pickle.dump(opt, ouf)
torch.save(F_s.state_dict(),
'{}/netF_s.pth'.format(opt.exp3_model_save_file))
for d in opt.domains:
if d in F_d:
torch.save(
F_d[d].state_dict(),
'{}/net_F_d_{}.pth'.format(opt.exp3_model_save_file,
d))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.exp3_model_save_file))
torch.save(D.state_dict(),
'{}/netD.pth'.format(opt.exp3_model_save_file))
# end of training
log.info(f'Best average validation accuracy: {best_avg_acc}')
log.info(
f'Loading model for feature visualization from {opt.exp3_model_save_file}...'
)
for domain in opt.domains:
F_d[domain].load_state_dict(
torch.load(
os.path.join(opt.exp3_model_save_file,
f'net_F_d_{domain}.pth')))
num_iter = len(train_loaders[opt.domains[0]])
# visual_features暂时不加上shared feature
# visual_features, senti_labels = get_visual_features(num_iter, unlabeled_loaders, unlabeled_iters, F_s, F_d)
visual_features, senti_labels = get_visual_features(
num_iter, unlabeled_loaders, unlabeled_iters, F_d)
return best_acc, visual_features, senti_labels
def train(vocab, train_sets, dev_sets, test_sets, unlabeled_sets):
train_loaders, unlabeled_loaders = {}, {}
train_iters, unlabeled_iters = {}, {}
dev_loaders, test_loaders = {}, {}
my_collate = utils.sorted_collate if opt.model == 'lstm' else utils.unsorted_collate
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size,
shuffle=True,
collate_fn=my_collate)
train_iters[domain] = iter(train_loaders[domain])
for domain in opt.dev_domains:
dev_loaders[domain] = DataLoader(dev_sets[domain],
opt.batch_size,
shuffle=False,
collate_fn=my_collate)
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size,
shuffle=False,
collate_fn=my_collate)
for domain in opt.all_domains:
if domain in opt.unlabeled_domains:
uset = unlabeled_sets[domain]
else:
# for labeled domains, consider which data to use as unlabeled set
if opt.unlabeled_data == 'both':
uset = ConcatDataset(
[train_sets[domain], unlabeled_sets[domain]])
elif opt.unlabeled_data == 'unlabeled':
uset = unlabeled_sets[domain]
elif opt.unlabeled_data == 'train':
uset = train_sets[domain]
else:
raise Exception(
'Unknown options for the unlabeled data usage: {}'.format(
opt.unlabeled_data))
unlabeled_loaders[domain] = DataLoader(uset,
opt.batch_size,
shuffle=True,
collate_fn=my_collate)
unlabeled_iters[domain] = iter(unlabeled_loaders[domain])
# models
F_s = None
C = None
if opt.model.lower() == 'dan':
F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
elif opt.model.lower() == 'lstm':
F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
elif opt.model.lower() == 'cnn':
F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.kernel_num, opt.kernel_sizes,
opt.dropout)
else:
raise Exception('Unknown model architecture {}'.format(opt.model))
C = SentimentClassifier(opt.C_layers,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.num_labels, opt.dropout, opt.C_bn)
F_s, C = F_s.to(opt.device), C.to(opt.device)
optimizer = optim.Adam(itertools.chain(
*map(list, [F_s.parameters() if F_s else [],
C.parameters()])),
lr=opt.learning_rate)
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# F&C iteration
utils.unfreeze_net(F_s)
utils.unfreeze_net(C)
if opt.fix_emb:
utils.freeze_net(F_s.word_emb)
F_s.zero_grad()
C.zero_grad()
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
shared_feat = F_s(inputs)
domain_feat = torch.zeros(
len(targets), opt.domain_hidden_size).to(opt.device)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join(
[str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain], F_s, None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_s,
None, C)
avg_test_acc = sum([test_acc[d]
for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
if avg_acc > best_avg_acc:
log.info(
'New best average validation accuracy: {}'.format(avg_acc))
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'),
'wb') as ouf:
pickle.dump(opt, ouf)
torch.save(F_s.state_dict(),
'{}/netF_s.pth'.format(opt.model_save_file))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.model_save_file))
# end of training
log.info('Best average validation accuracy: {}'.format(best_avg_acc))
return best_acc
def train(train_sets, test_sets):
# ---------------------- dataloader ---------------------- #
# dataset loaders
train_loaders, train_iters, test_loaders, test_iters = {}, {}, {}, {}
# 加载有label的训练数据
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size,
shuffle=True)
train_iters[domain] = iter(train_loaders[domain])
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size,
shuffle=False)
test_iters[domain] = iter(test_loaders[domain])
# ---------------------- model initialization ---------------------- #
F_d = {}
C = None
if opt.model.lower() == 'mlp':
for domain in opt.domains:
F_d[domain] = MlpFeatureExtractor(opt.feature_num,
opt.F_hidden_sizes,
opt.domain_hidden_size,
opt.dropout, opt.F_bn)
C = SentimentClassifier(opt.C_layers, opt.domain_hidden_size,
opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
# 转移到gpu上
C = C.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
criterion_cent = CenterLoss(num_classes=2,
feat_dim=opt.domain_hidden_size,
use_gpu=True)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=0.5)
optimizer = optim.Adam(itertools.chain(
*map(list, [C.parameters()] + [f.parameters() for f in F_d.values()])),
lr=opt.learning_rate)
# training
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
best_acc = 0.0
best_acc_dict = {}
margin = 3
margin_lambda = 0.1
# center_loss_weight_cent = 0.1
for epoch in range(opt.max_epoch):
C.train()
for f in F_d.values():
f.train()
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
# First_stage
for _ in tqdm(range(num_iter)):
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
optimizer_centloss.zero_grad()
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
domain_feat = F_d[domain](inputs)
visual_feature, c_outputs = C(domain_feat)
# loss_cent = criterion_cent(visual_feature, targets)
# loss_cent *= center_loss_weight_cent
loss_cent = 0.0
loss_part_1 = functional.nll_loss(c_outputs, targets)
targets = targets.unsqueeze(1)
targets_onehot = torch.FloatTensor(opt.batch_size, 2)
targets_onehot.zero_()
targets_onehot.scatter_(1, targets.cpu(), 1)
targets_onehot = targets_onehot.to(opt.device)
loss_part_2 = margin_lambda * margin_regularization(
inputs, targets_onehot, F_d[domain], C, margin)
# loss_part_2 = 0.0
print("loss_part_1: " + str(loss_part_1))
print("loss_part_2: " + str(loss_part_2))
print("loss_cent: " + str(loss_cent))
l_c = loss_part_1 + loss_part_2 + loss_cent
l_c.backward()
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
optimizer.step()
# for param in criterion_cent.parameters():
# param.grad.data *= (1. / center_loss_weight_cent)
optimizer_centloss.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct /
d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join(
[str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_d[domain],
C)
avg_test_acc = sum([test_acc[d] for d in opt.domains]) / len(opt.domains)
log.info(f'Average test accuracy: {avg_test_acc}')
if avg_test_acc > best_acc:
log.info(f'New best Average test accuracy: {avg_test_acc}')
best_acc = avg_test_acc
best_acc_dict = test_acc
for d in opt.domains:
if d in F_d:
torch.save(
F_d[d].state_dict(),
'{}/net_F_d_{}.pth'.format(opt.exp2_model_save_file, d))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.exp2_model_save_file))
log.info(
f'Loading model for feature visualization from {opt.exp2_model_save_file}...'
)
for domain in opt.domains:
F_d[domain].load_state_dict(
torch.load(
os.path.join(opt.exp2_model_save_file,
f'net_F_d_{domain}.pth')))
num_iter = len(train_loaders[opt.domains[0]])
visual_features, senti_labels = get_visual_features(
num_iter, test_loaders, test_iters, F_d, C)
# visual_features, senti_labels = get_visual_features(num_iter, train_loaders, train_iters, F_d)
return best_acc, best_acc_dict, visual_features, senti_labels
def train_nobert(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders):
# models
F_s = None
F_d = {}
C, D = None, None
if opt.model.lower() == 'dan':
F_s = DanFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
for domain in opt.domains:
F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
elif opt.model.lower() == 'lstm':
F_s = LSTMFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
for domain in opt.domains:
F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
elif opt.model.lower() == 'cnn':
F_s = CNNFeatureExtractor(vocab, opt.F_layers, opt.shared_hidden_size,
opt.kernel_num, opt.kernel_sizes, opt.dropout)
for domain in opt.domains:
F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.kernel_num, opt.kernel_sizes, opt.dropout)
else:
raise Exception('Unknown model architecture {}'.format(opt.model))
C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
D = DomainClassifier(opt.D_layers, opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn)
F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(*map(list, [F_s.parameters() if F_s else [], C.parameters()] + [f.parameters() for f in F_d.values()])), lr=opt.learning_rate)
optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate)
# testing
if opt.test_only:
log.info('Loading model from {}...'.format(opt.model_save_file))
if F_s:
F_s.load_state_dict(torch.load(os.path.join(opt.model_save_file,
'netF_s.pth')))
for domain in opt.all_domains:
if domain in F_d:
F_d[domain].load_state_dict(torch.load(os.path.join(opt.model_save_file,
'net_F_d_{}.pth'.format(domain))))
C.load_state_dict(torch.load(os.path.join(opt.model_save_file,
'netC.pth')))
D.load_state_dict(torch.load(os.path.join(opt.model_save_file,
'netD.pth')))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.all_domains:
acc[domain] = evaluate(domain, dev_loaders[domain],
F_s, F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.all_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain],
F_s, F_d[domain] if domain in F_d else None, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
return {'valid': acc, 'test': test_acc}
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
D.train()
for f in F_d.values():
f.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# D iterations
utils.freeze_net(F_s)
map(utils.freeze_net, F_d.values())
utils.freeze_net(C)
utils.unfreeze_net(D)
# WGAN n_critic trick since D trains slower
n_critic = opt.n_critic
if opt.wgan_trick:
if opt.n_critic>0 and ((epoch==0 and i<25) or i%500==0):
n_critic = 100
for _ in range(n_critic):
D.zero_grad()
loss_d = {}
# train on both labeled and unlabeled domains
for domain in opt.all_domains:
# targets not used
batch = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
batch = tuple(t.to(opt.device) for t in batch)
emb_ids, input_ids, input_mask, segment_ids, label_ids = batch
d_targets = utils.get_domain_label(opt.loss, domain, len(emb_ids))
shared_feat = F_s(emb_ids)
d_outputs = D(shared_feat)
# D accuracy
_, pred = torch.max(d_outputs, 1)
d_total += len(emb_ids)
if opt.loss.lower() == 'l2':
_, tgt_indices = torch.max(d_targets, 1)
d_correct += (pred==tgt_indices).sum().item()
l_d = functional.mse_loss(d_outputs, d_targets)
l_d.backward()
else:
d_correct += (pred==d_targets).sum().item()
l_d = functional.nll_loss(d_outputs, d_targets)
l_d.backward()
loss_d[domain] = l_d.item()
optimizerD.step()
# F&C iteration
utils.unfreeze_net(F_s)
map(utils.unfreeze_net, F_d.values())
utils.unfreeze_net(C)
utils.freeze_net(D)
# if opt.fix_emb:
# utils.freeze_net(F_s.word_emb)
# for f_d in F_d.values():
# utils.freeze_net(f_d.word_emb)
F_s.zero_grad()
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
for domain in opt.domains:
batch = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
batch = tuple(t.to(opt.device) for t in batch)
emb_ids, input_ids, input_mask, segment_ids, label_ids = batch
inputs = emb_ids
targets = label_ids
shared_feat = F_s(inputs)
domain_feat = F_d[domain](inputs)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
# update F with D gradients on all domains
for domain in opt.all_domains:
batch = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
batch = tuple(t.to(opt.device) for t in batch)
emb_ids, input_ids, input_mask, segment_ids, label_ids = batch
d_inputs = emb_ids
shared_feat = F_s(d_inputs)
d_outputs = D(shared_feat)
if opt.loss.lower() == 'gr':
d_targets = utils.get_domain_label(opt.loss, domain, len(d_inputs))
l_d = functional.nll_loss(d_outputs, d_targets)
if opt.lambd > 0:
l_d *= -opt.lambd
elif opt.loss.lower() == 'bs':
d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs))
l_d = functional.kl_div(d_outputs, d_targets, size_average=False)
if opt.lambd > 0:
l_d *= opt.lambd
elif opt.loss.lower() == 'l2':
d_targets = utils.get_random_domain_label(opt.loss, len(d_inputs))
l_d = functional.mse_loss(d_outputs, d_targets)
if opt.lambd > 0:
l_d *= opt.lambd
l_d.backward()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch+1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0*d_correct/d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join([str(100.0*correct[d]/total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain],
F_s, F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain],
F_s, F_d[domain] if domain in F_d else None, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
if avg_acc > best_avg_acc:
log.info('New best average validation accuracy: {}'.format(avg_acc))
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf:
pickle.dump(opt, ouf)
# torch.save(F_s.state_dict(),
# '{}/netF_s.pth'.format(opt.model_save_file))
# for d in opt.domains:
# if d in F_d:
# torch.save(F_d[d].state_dict(),
# '{}/net_F_d_{}.pth'.format(opt.model_save_file, d))
# torch.save(C.state_dict(),
# '{}/netC.pth'.format(opt.model_save_file))
# torch.save(D.state_dict(),
# '{}/netD.pth'.format(opt.model_save_file))
# end of training
log.info('Best average validation accuracy: {}'.format(best_avg_acc))
return best_acc
def train_private_nobert(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders):
# models
F_d = {}
C = None
if opt.model.lower() == 'dan':
for domain in opt.domains:
F_d[domain] = DanFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.sum_pooling, opt.dropout, opt.F_bn)
elif opt.model.lower() == 'lstm':
for domain in opt.domains:
F_d[domain] = LSTMFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.dropout, opt.bdrnn, opt.attn)
elif opt.model.lower() == 'cnn':
for domain in opt.domains:
F_d[domain] = CNNFeatureExtractor(vocab, opt.F_layers, opt.domain_hidden_size,
opt.kernel_num, opt.kernel_sizes, opt.dropout)
else:
raise Exception('Unknown model architecture {}'.format(opt.model))
C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
C = C.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(
*map(list, [[], C.parameters()] + [f.parameters() for f in F_d.values()])),
lr=opt.learning_rate)
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
C.train()
for f in F_d.values():
f.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# F&C iteration
map(utils.unfreeze_net, F_d.values())
utils.unfreeze_net(C)
if opt.fix_emb:
for f_d in F_d.values():
utils.freeze_net(f_d.word_emb)
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
for domain in opt.domains:
batch = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
batch = tuple(t.to(opt.device) for t in batch)
emb_ids, input_ids, input_mask, segment_ids, label_ids = batch
inputs = emb_ids
targets = label_ids
shared_feat = torch.zeros(len(targets), opt.shared_hidden_size).to(opt.device)
domain_feat = F_d[domain](inputs)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain],
None, F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain],
None, F_d[domain] if domain in F_d else None, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
if avg_acc > best_avg_acc:
log.info('New best average validation accuracy: {}'.format(avg_acc))
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf:
pickle.dump(opt, ouf)
# for d in opt.domains:
# if d in F_d:
# torch.save(F_d[d].state_dict(),
# '{}/net_F_d_{}.pth'.format(opt.model_save_file, d))
# torch.save(C.state_dict(),
# '{}/netC.pth'.format(opt.model_save_file))
# end of training
log.info('Best average validation accuracy: {}'.format(best_avg_acc))
return best_acc
best_dev_f = -10
bad_counter = 0
for epoch in range(opt.max_epoch):
model.train()
optimizer.zero_grad()
# sum_cost = 0.0
correct, total = 0, 0
epoch_start = time.time()
for i in range(num_iter):
mention_inputs, _, sentences, mask, targets = utils.endless_get_next_batch(train_loader, train_iter)
batch_output = model(mention_inputs, sentences, mask)
cost = criterion(batch_output, targets)
# sum_cost += cost.item()
# train accuracy
total += targets.size(0)
_, pred = torch.max(batch_output, 1)
# correct += (pred == targets).sum().sample_data[0]
correct += (pred == targets).sum().item()
cost.backward()
optimizer.step()
optimizer.zero_grad()
def train_shared(vocab, train_loaders, unlabeled_loaders, train_iters, unlabeled_iters, dev_loaders, test_loaders, F_s):
C = None
C = SentimentClassifier(opt.C_layers, opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
F_s, C = F_s.to(opt.device), C.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(
*map(list, [F_s.parameters() if F_s else [], C.parameters()] + [])),
lr=opt.learning_rate)
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# F&C iteration
utils.unfreeze_net(F_s)
utils.unfreeze_net(C)
if opt.fix_emb:
utils.freeze_net(F_s.word_emb)
F_s.zero_grad()
C.zero_grad()
for domain in opt.domains:
batch = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
batch = tuple(t.to(opt.device) for t in batch)
emb_ids, input_ids, input_mask, segment_ids, label_ids = batch
inputs = emb_ids
targets = label_ids
_, shared_feat = F_s(input_ids, input_mask, segment_ids)
domain_feat = torch.zeros(len(targets), opt.domain_hidden_size).to(opt.device)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct / d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join([str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.dev_domains:
acc[domain] = evaluate(domain, dev_loaders[domain],
F_s, None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average validation accuracy: {}'.format(avg_acc))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.dev_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain],
F_s, None, C)
avg_test_acc = sum([test_acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info('Average test accuracy: {}'.format(avg_test_acc))
if avg_acc > best_avg_acc:
log.info('New best average validation accuracy: {}'.format(avg_acc))
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'), 'wb') as ouf:
pickle.dump(opt, ouf)
# torch.save(C.state_dict(),
# '{}/netC.pth'.format(opt.model_save_file))
# end of training
log.info('Best average validation accuracy: {}'.format(best_avg_acc))
return best_acc
def train(train_sets, dev_sets, test_sets, unlabeled_sets, fold):
"""
train_sets, dev_sets, test_sets: dict[domain] -> TensorDataset
For unlabeled domains, no train_sets are available
"""
# dataset loaders
train_loaders, unlabeled_loaders = {}, {}
train_iters, unlabeled_iters = {}, {}
dev_loaders, test_loaders = {}, {}
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size,
shuffle=True)
train_iters[domain] = iter(train_loaders[domain])
for domain in opt.all_domains:
dev_loaders[domain] = DataLoader(dev_sets[domain],
opt.batch_size,
shuffle=False)
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size,
shuffle=False)
if domain in opt.unlabeled_domains:
uset = unlabeled_sets[domain]
else:
# for labeled domains, consider which data to use as unlabeled set
if opt.unlabeled_data == 'both':
uset = ConcatDataset(
[train_sets[domain], unlabeled_sets[domain]])
elif opt.unlabeled_data == 'unlabeled':
uset = unlabeled_sets[domain]
elif opt.unlabeled_data == 'train':
uset = train_sets[domain]
else:
raise Exception(
f'Unknown options for the unlabeled data usage: {opt.unlabeled_data}'
)
unlabeled_loaders[domain] = DataLoader(uset,
opt.batch_size,
shuffle=True)
unlabeled_iters[domain] = iter(unlabeled_loaders[domain])
# models
F_s = None
F_d = {}
C, D = None, None
if opt.model.lower() == 'mlp':
F_s = MlpFeatureExtractor(opt.feature_num, opt.F_hidden_sizes,
opt.shared_hidden_size, opt.dropout,
opt.F_bn)
for domain in opt.domains:
F_d[domain] = MlpFeatureExtractor(opt.feature_num,
opt.F_hidden_sizes,
opt.domain_hidden_size,
opt.dropout, opt.F_bn)
else:
raise Exception(f'Unknown model architecture {opt.model}')
C = SentimentClassifier(opt.C_layers,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.shared_hidden_size + opt.domain_hidden_size,
opt.num_labels, opt.dropout, opt.C_bn)
D = DomainClassifier(opt.D_layers,
opt.shared_hidden_size, opt.shared_hidden_size,
len(opt.all_domains), opt.loss, opt.dropout, opt.D_bn)
F_s, C, D = F_s.to(opt.device), C.to(opt.device), D.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
# optimizers
optimizer = optim.Adam(itertools.chain(
*map(list, [F_s.parameters() if F_s else [],
C.parameters()] + [f.parameters() for f in F_d.values()])),
lr=opt.learning_rate)
optimizerD = optim.Adam(D.parameters(), lr=opt.D_learning_rate)
# testing
if opt.test_only:
log.info(f'Loading model from {opt.model_save_file}...')
F_s.load_state_dict(
torch.load(
os.path.join(opt.model_save_file, f'netF_s_fold{fold}.pth')))
for domain in opt.all_domains:
if domain in F_d:
F_d[domain].load_state_dict(
torch.load(
os.path.join(opt.model_save_file,
f'net_F_d_{domain}_fold{fold}.pth')))
C.load_state_dict(
torch.load(
os.path.join(opt.model_save_file, f'netC_fold{fold}.pth')))
D.load_state_dict(
torch.load(
os.path.join(opt.model_save_file, f'netD_fold{fold}.pth')))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.all_domains:
acc[domain] = evaluate(domain, dev_loaders[domain], F_s,
F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average validation accuracy: {avg_acc}')
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.all_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_s,
F_d[domain] if domain in F_d else None,
C)
avg_test_acc = sum([test_acc[d]
for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average test accuracy: {avg_test_acc}')
return {'valid': acc, 'test': test_acc}
# training
best_acc, best_avg_acc = defaultdict(float), 0.0
for epoch in range(opt.max_epoch):
F_s.train()
C.train()
D.train()
for f in F_d.values():
f.train()
# training accuracy
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
for i in tqdm(range(num_iter)):
# D iterations
utils.freeze_net(F_s)
map(utils.freeze_net, F_d.values())
utils.freeze_net(C)
utils.unfreeze_net(D)
# optional WGAN n_critic trick
n_critic = opt.n_critic
if opt.wgan_trick:
if opt.n_critic > 0 and ((epoch == 0 and i < 25)
or i % 500 == 0):
n_critic = 100
for _ in range(n_critic):
D.zero_grad()
loss_d = {}
# train on both labeled and unlabeled domains
for domain in opt.all_domains:
# targets not used
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
d_targets = utils.get_domain_label(opt.loss, domain,
len(d_inputs))
shared_feat = F_s(d_inputs)
d_outputs = D(shared_feat)
# D accuracy
_, pred = torch.max(d_outputs, 1)
d_total += len(d_inputs)
if opt.loss.lower() == 'l2':
_, tgt_indices = torch.max(d_targets, 1)
d_correct += (pred == tgt_indices).sum().item()
l_d = functional.mse_loss(d_outputs, d_targets)
l_d.backward()
else:
d_correct += (pred == d_targets).sum().item()
l_d = functional.nll_loss(d_outputs, d_targets)
l_d.backward()
loss_d[domain] = l_d.item()
optimizerD.step()
# F&C iteration
utils.unfreeze_net(F_s)
map(utils.unfreeze_net, F_d.values())
utils.unfreeze_net(C)
utils.freeze_net(D)
F_s.zero_grad()
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
shared_feats, domain_feats = [], []
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
shared_feat = F_s(inputs)
shared_feats.append(shared_feat)
domain_feat = F_d[domain](inputs)
domain_feats.append(domain_feat)
features = torch.cat((shared_feat, domain_feat), dim=1)
c_outputs = C(features)
l_c = functional.nll_loss(c_outputs, targets)
l_c.backward(retain_graph=True)
# training accuracy
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
# update F with D gradients on all domains
for domain in opt.all_domains:
d_inputs, _ = utils.endless_get_next_batch(
unlabeled_loaders, unlabeled_iters, domain)
shared_feat = F_s(d_inputs)
d_outputs = D(shared_feat)
if opt.loss.lower() == 'gr':
d_targets = utils.get_domain_label(opt.loss, domain,
len(d_inputs))
l_d = functional.nll_loss(d_outputs, d_targets)
log.debug(f'D loss: {l_d.item()}')
if opt.lambd > 0:
l_d *= -opt.lambd
elif opt.loss.lower() == 'bs':
d_targets = utils.get_random_domain_label(
opt.loss, len(d_inputs))
l_d = functional.kl_div(d_outputs,
d_targets,
size_average=False)
if opt.lambd > 0:
l_d *= opt.lambd
elif opt.loss.lower() == 'l2':
d_targets = utils.get_random_domain_label(
opt.loss, len(d_inputs))
l_d = functional.mse_loss(d_outputs, d_targets)
if opt.lambd > 0:
l_d *= opt.lambd
l_d.backward()
if opt.model.lower() != 'lstm':
log.debug(
f'F_s norm: {F_s.net[-2].weight.grad.data.norm()}')
optimizer.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct /
d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join(
[str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating validation sets:')
acc = {}
for domain in opt.all_domains:
acc[domain] = evaluate(domain, dev_loaders[domain], F_s,
F_d[domain] if domain in F_d else None, C)
avg_acc = sum([acc[d] for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average validation accuracy: {avg_acc}')
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.all_domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_s,
F_d[domain] if domain in F_d else None,
C)
avg_test_acc = sum([test_acc[d]
for d in opt.dev_domains]) / len(opt.dev_domains)
log.info(f'Average test accuracy: {avg_test_acc}')
if avg_acc > best_avg_acc:
log.info(f'New best average validation accuracy: {avg_acc}')
best_acc['valid'] = acc
best_acc['test'] = test_acc
best_avg_acc = avg_acc
with open(os.path.join(opt.model_save_file, 'options.pkl'),
'wb') as ouf:
pickle.dump(opt, ouf)
torch.save(
F_s.state_dict(),
'{}/netF_s_fold{}.pth'.format(opt.model_save_file, fold))
for d in opt.domains:
if d in F_d:
torch.save(
F_d[d].state_dict(), '{}/net_F_d_{}_fold{}.pth'.format(
opt.model_save_file, d, fold))
torch.save(C.state_dict(),
'{}/netC_fold{}.pth'.format(opt.model_save_file, fold))
torch.save(D.state_dict(),
'{}/netD_fold{}.pth'.format(opt.model_save_file, fold))
# end of training
log.info(f'Best average validation accuracy: {best_avg_acc}')
return best_acc
