def test(classifier, generator, data_loader, dataset="MNIST"):
"""Evaluate classifier on source or target domains."""
# set eval state for Dropout and BN layers
generator.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels.squeeze_())
preds = classifier(generator(images))
loss += criterion(preds, labels).data[0]
pred_cls = preds.data.max(1)[1]
acc += pred_cls.eq(labels.data).cpu().sum()
loss /= len(data_loader)
acc /= len(data_loader.dataset)
print("Avg Loss = {:.5f}, Avg Accuracy = {:2.5%}".format(loss, acc))
def evaluate_target(encoder, classifier, images, labels):
"""Evaluate classifier on source or target domains."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
images = make_variable(images, volatile=True)
# labels = make_variable(labels.squeeze_())
labels = make_variable(labels).long().squeeze()
preds = classifier(encoder(images))
_, target_predicted = torch.max(preds.data, 1)
target_correct = (target_predicted == labels.data).sum()
acc = target_correct / labels.size()[0]
return acc
def eval_func_datacollect(encoder, classifier, data_loader):
"""Evaluate classifier for source domain."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
labelsall = []
predsall = []
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
# labels[labels == 10] = 0
labels = make_variable(labels).long().squeeze()
preds = classifier(encoder(images))
pred_cls = preds.data.max(1)[1]
labelsall.extend(labels.data.cpu().numpy())
predsall.extend(pred_cls.cpu().numpy())
return labelsall, predsall
def train_src_rec(encoder, classifier, generator, data_loader):
"""Train classifier for source domain."""
####################
# 1. setup network #
####################
# set train state for Dropout and BN layers
encoder.train()
classifier.train()
generator.train()
# setup criterion and optimizer
optimizer = optim.Adam(
generator.parameters(),
lr=cfg.learning_rate_apt,
betas=(cfg.beta1, cfg.beta2))
criterionRec = torch.nn.MSELoss()
####################
# 2. train network #
####################
for epoch in range(cfg.num_epochs_pre_rec):
for step, (images, labels) in enumerate(data_loader):
# make images and labels variable
images = make_variable(images)
# labels[labels == 10] = 0
labels = make_variable(labels).long().squeeze()
# zero gradients for optimizer
optimizer.zero_grad()
# compute loss for critic
feat= encoder(images)
feat_reshape = (feat.unsqueeze(2)).unsqueeze(2)
reconst = generator(feat_reshape)
loss_rec = criterionRec(reconst, images)
loss_rec.backward()
optimizer.step()
# print step info
if (step+1) % cfg.log_step_pre == 0:
print('Epoch [%d] '
'loss[%.2f] '
%(epoch,
loss_rec.data[0],
)
)
# # save final model
save_model(generator, "ADDA-source-generator-final.pt")
return generator
def evaluate(encoder, classifier, data_loader):
"""Evaluate classifier on source or target domains."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0.0
acc = 0.0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels.squeeze_())
#print('images shape = {}'.format(images.shape))
#print('encoding shape = {}'.format(encoder(images).shape))
preds = classifier(encoder(images))
#print('preds shape = {}'.format(preds.shape))
#print('labels shape = {}'.format(labels.shape))
cv = criterion(preds, labels)
#print('crit val = {}'.format(cv))
loss += cv.item() #data[0]
pred_cls = preds.data.max(1)[1]
#print('labels.data = {}, preds_cls = {}'.format(labels.data, pred_cls)) #!!
#print('eq = {}'.format(pred_cls.eq(labels.data)))
contrib = pred_cls.eq(labels.data).cpu().sum()
#print('contrib = {}'.format(contrib))
acc += float(contrib)
loss /= float(len(data_loader))
acc /= float(len(data_loader.dataset))
print("Avg Loss = {:.5f}, Avg Accuracy = {:2.5%}".format(loss, acc))
def eval_func(encoder, classifier, data_loader, sample=False):
"""Evaluate classifier for source domain."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
# labels[labels == 10] = 0
labels = make_variable(labels).long().squeeze()
preds = classifier(encoder(images))
loss += criterion(preds, labels).item()
pred_cls = preds.data.max(1)[1]
acc += pred_cls.eq(labels.data).cpu().sum().item()
loss /= len(data_loader)
acc /= len(data_loader.dataset)
# acc /= 1800
# acc /= 2000
# if sample:
# # acc /= (len(data_loader)*cfg.batch_size)
# acc /= 1800
# # acc /= 2000
# else:
# # acc /= 1800
# # acc /= 2000
# acc /= len(data_loader.dataset)
print("Avg Loss = {}, Avg Accuracy = {:2%}".format(loss, acc))
def extract(model, filepath, vid):
"""Extract features by inception_v3."""
# data loader for frames in ingle video
data_loader = get_dataloader(dataset="VideoFrame",
path=filepath,
num_frames=cfg.num_frames,
batch_size=cfg.batch_size)
# extract features by inception_v3
feats = None
for step, frames in enumerate(data_loader):
print("--> extract features [{}/{}]".format(step + 1,
len(data_loader)))
feat = model(make_variable(frames))
feats = concat_feat_var(feats, feat.data.cpu())
print("--> save feats to {}".format(
cfg.inception_v3_feats_path.format(vid)))
torch.save(feats, cfg.inception_v3_feats_path.format(vid))
print("--> delete original video file: {}".format(filepath))
os.remove(filepath)
def genarate_labels(F, F_1, F_2, target_dataset, num_target):
"""Genrate pseudo labels for target domain dataset."""
# set eval state for Dropout and BN layers
F.eval()
F_1.eval()
F_2.eval()
# get candidate samples
print("Num of sampled target data: {}".format(num_target))
# get sampled data loader
data_loader = get_sampled_data_loader(target_dataset,
num_target,
shuffle=True)
# get output of F_1 and F_2 on sampled target dataset
out_F_1_total = None
out_F_2_total = None
for step, (images, _) in enumerate(data_loader):
# convert into torch.autograd.Variable
images = make_variable(images)
# forward networks
out_F = F(images)
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
# concat outputs
if step == 0:
out_F_1_total = out_F_1.data.cpu()
out_F_2_total = out_F_2.data.cpu()
else:
out_F_1_total = torch.cat([out_F_1_total, out_F_1.data.cpu()], 0)
out_F_2_total = torch.cat([out_F_2_total, out_F_2.data.cpu()], 0)
# guess pseudo labels
excerpt, pseudo_labels = \
guess_pseudo_labels(out_F_1_total, out_F_2_total)
return excerpt, pseudo_labels
def train_src(encoder, classifier, data_loader, tgt_data_loader_eval):
"""Train classifier for source domain."""
####################
# 1. setup network #
####################
# set train state for Dropout and BN layers
encoder.train()
classifier.train()
# setup criterion and optimizer
optimizer = optim.Adam(list(encoder.parameters()) +
list(classifier.parameters()),
lr=cfg.learning_rate_pre,
betas=(cfg.beta1, cfg.beta2))
criterion = nn.CrossEntropyLoss()
# optimizer = optim.Adam(
# itertools(encoder.parameters(), classifier.parameters()),
# lr=cfg.learning_rate_pre,
# betas=(cfg.beta1, cfg.beta2))
# criterion = nn.CrossEntropyLoss()
####################
# 2. train network #
####################
for epoch in range(cfg.num_epochs_pre):
for step, (images, labels) in enumerate(data_loader):
# make images and labels variable
images = make_variable(images)
# labels[labels == 10] = 0
labels = make_variable(labels).long().squeeze()
# zero gradients for optimizer
optimizer.zero_grad()
# compute loss for critic
preds = classifier(encoder(images))
loss = criterion(preds, labels)
# optimize source classifier
loss.backward()
optimizer.step()
# st()
acc = evaluate.evaluate_step(encoder, classifier, images, labels)
# print step info
if (step + 1) % cfg.log_step_pre == 0:
print('Epoch [%d] '
'loss[%.2f] '
'Source_Accuracy[%.2f] ' % (epoch, loss.data[0], acc))
# eval model on test set
if ((epoch + 1) % cfg.eval_step_pre == 0):
# evaluate.eval_func(encoder, classifier, tgt_data_loader_eval, sample=True)
# evaluate.eval_func(encoder, classifier, data_loader)
print(">>> source only <<<")
evaluate.eval_func(encoder, classifier, tgt_data_loader_eval)
# save model parameters
if ((epoch + 1) % cfg.save_step_pre == 0):
save_model(encoder, "ADDA-source-encoder-{}.pt".format(epoch + 1))
save_model(classifier,
"ADDA-source-classifier-{}.pt".format(epoch + 1))
# # save final model
save_model(encoder, "ADDA-source-encoder-final.pt")
save_model(classifier, "ADDA-source-classifier-final.pt")
return encoder, classifier
def train(classifier, generator, critic, src_data_loader, tgt_data_loader):
"""Train generator, classifier and critic jointly."""
####################
# 1. setup network #
####################
# set train state for Dropout and BN layers
classifier.train()
generator.train()
critic.train()
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
optimizer_c = get_optimizer(classifier, "Adam")
optimizer_g = get_optimizer(generator, "Adam")
optimizer_d = get_optimizer(critic, "Adam")
# zip source and target data pair
data_iter_src = get_inf_iterator(src_data_loader)
data_iter_tgt = get_inf_iterator(tgt_data_loader)
# counter
g_step = 0
# positive and negative labels
pos_labels = make_variable(torch.FloatTensor([1]))
neg_labels = make_variable(torch.FloatTensor([-1]))
####################
# 2. train network #
####################
for epoch in range(params.num_epochs):
###########################
# 2.1 train discriminator #
###########################
# requires to compute gradients for D
for p in critic.parameters():
p.requires_grad = True
# set steps for discriminator
if g_step < 25 or g_step % 500 == 0:
# this helps to start with the critic at optimum
# even in the first iterations.
critic_iters = 100
else:
critic_iters = params.d_steps
# loop for optimizing discriminator
for d_step in range(critic_iters):
# convert images into torch.Variable
images_src, labels_src = next(data_iter_src)
images_tgt, _ = next(data_iter_tgt)
images_src = make_variable(images_src)
labels_src = make_variable(labels_src.squeeze_())
images_tgt = make_variable(images_tgt)
if images_src.size(0) != params.batch_size or \
images_tgt.size(0) != params.batch_size:
continue
# zero gradients for optimizer
optimizer_d.zero_grad()
# compute source data loss for discriminator
feat_src = generator(images_src)
d_loss_src = critic(feat_src.detach())
d_loss_src = d_loss_src.mean()
d_loss_src.backward(neg_labels)
# compute target data loss for discriminator
feat_tgt = generator(images_tgt)
d_loss_tgt = critic(feat_tgt.detach())
d_loss_tgt = d_loss_tgt.mean()
d_loss_tgt.backward(pos_labels)
# compute gradient penalty
gradient_penalty = calc_gradient_penalty(critic, feat_src.data,
feat_tgt.data)
gradient_penalty.backward()
# optimize weights of discriminator
d_loss = -d_loss_src + d_loss_tgt + gradient_penalty
optimizer_d.step()
########################
# 2.2 train classifier #
########################
# zero gradients for optimizer
optimizer_c.zero_grad()
# compute loss for critic
preds_c = classifier(generator(images_src).detach())
c_loss = criterion(preds_c, labels_src)
# optimize source classifier
c_loss.backward()
optimizer_c.step()
#######################
# 2.3 train generator #
#######################
# avoid to compute gradients for D
for p in critic.parameters():
p.requires_grad = False
# zero grad for optimizer of generator
optimizer_g.zero_grad()
# compute source data classification loss for generator
feat_src = generator(images_src)
preds_c = classifier(feat_src)
g_loss_cls = criterion(preds_c, labels_src)
g_loss_cls.backward()
# compute source data discriminattion loss for generator
feat_src = generator(images_src)
g_loss_src = critic(feat_src).mean()
g_loss_src.backward(pos_labels)
# compute target data discriminattion loss for generator
feat_tgt = generator(images_tgt)
g_loss_tgt = critic(feat_tgt).mean()
g_loss_tgt.backward(neg_labels)
# compute loss for generator
g_loss = g_loss_src - g_loss_tgt + g_loss_cls
# optimize weights of generator
optimizer_g.step()
g_step += 1
##################
# 2.4 print info #
##################
if ((epoch + 1) % params.log_step == 0):
print("Epoch [{}/{}]:"
"d_loss={:.5f} c_loss={:.5f} g_loss={:.5f} "
"D(x)={:.5f} D(G(z))={:.5f} GP={:.5f}".format(
epoch + 1, params.num_epochs, d_loss.data[0],
c_loss.data[0], g_loss.data[0], d_loss_src.data[0],
d_loss_tgt.data[0], gradient_penalty.data[0]))
#############################
# 2.5 save model parameters #
#############################
if ((epoch + 1) % params.save_step == 0):
save_model(critic, "WGAN-GP_critic-{}.pt".format(epoch + 1))
save_model(classifier,
"WGAN-GP_classifier-{}.pt".format(epoch + 1))
save_model(generator, "WGAN-GP_generator-{}.pt".format(epoch + 1))
return classifier, generator
def domain_adapt(F, F_1, F_2, F_t, source_dataset, target_dataset, excerpt,
pseudo_labels, plot):
"""Perform Doamin Adaptation between source and target domains."""
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
if 0:
optimType = "Adam"
cfg.learning_rate = 1.0E-4
else:
optimType = "sgd"
cfg.learning_rate = 1.0E-4
optimizer_F = get_optimizer(F, optimType)
optimizer_F_1 = get_optimizer(F_1, optimType)
optimizer_F_2 = get_optimizer(F_2, optimType)
optimizer_F_t = get_optimizer(F_t, optimType)
# get labelled target dataset
print('pseudo_labels = %s' % str(pseudo_labels))
target_dataset_labelled = get_dummy(target_dataset,
excerpt,
pseudo_labels,
get_dataset=True)
# merge soruce data and target data
merged_dataset = ConcatDataset([source_dataset, target_dataset_labelled])
print('target_dataset_labelled = %d' % len(target_dataset_labelled))
# start training
plt.figure()
for k in range(cfg.num_epochs_k):
# set train state for Dropout and BN layers
F.train()
F_1.train()
F_2.train()
F_t.train()
losses = []
merged_dataloader = make_data_loader(merged_dataset)
target_dataloader_labelled = make_data_loader(target_dataset_labelled)
target_dataloader_labelled_iter = get_inf_iterator(
target_dataloader_labelled)
if 0:
plt.figure()
atr.showDataSet(target_dataloader_labelled)
plt.waitforbuttonpress()
if 0:
# There's a bug here, the labels are not the same data type. print them out!!
source_dataloader_iter = get_inf_iterator(
make_data_loader(source_dataset))
a, b = next(source_dataloader_iter)
c, d = next(target_dataloader_labelled_iter)
print('source labels = {}'.format(b))
print('target labels = {}'.format(d))
sys.exit(0)
for epoch in range(cfg.num_epochs_adapt):
if optimType == 'sgd':
adjustLearningRate(optimizer_F, cfg.learning_rate, epoch,
cfg.num_epochs_adapt)
adjustLearningRate(optimizer_F_1, cfg.learning_rate, epoch,
cfg.num_epochs_adapt)
adjustLearningRate(optimizer_F_2, cfg.learning_rate, epoch,
cfg.num_epochs_adapt)
adjustLearningRate(optimizer_F_t, cfg.learning_rate, epoch,
cfg.num_epochs_adapt)
for step, rez in enumerate(merged_dataloader):
#!!print('rez = %s' % rez)
images, labels = rez
if images.shape[0] < cfg.batch_size:
print('WARNING: batch of size %d smaller than desired %d: skipping' % \
(images.shape[0], cfg.batch_size))
continue
# sample from T_l
images_tgt, labels_tgt = next(target_dataloader_labelled_iter)
while images_tgt.shape[0] < cfg.batch_size:
print('WARNING: target batch of size %d smaller than desired %d' % \
(images_tgt.shape[0], cfg.batch_size))
images_tgt, labels_tgt = next(
target_dataloader_labelled_iter)
# convert into torch.autograd.Variable
images = make_variable(images)
labels = make_variable(labels)
images_tgt = make_variable(images_tgt)
labels_tgt = make_variable(labels_tgt)
# zero-grad optimizer
optimizer_F.zero_grad()
optimizer_F_1.zero_grad()
optimizer_F_2.zero_grad()
optimizer_F_t.zero_grad()
# forward networks
#print('images shape = {}'.format(images.shape))#!!
out_F = F(images)
#print('out_F = {}'.format(out_F.shape))#!!
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
out_F_t = F_t(F(images_tgt))
# compute labelling loss
loss_similiar = calc_similiar_penalty(F_1, F_2)
loss_F_1 = criterion(out_F_1, labels)
loss_F_2 = criterion(out_F_2, labels)
loss_labelling = loss_F_1 + loss_F_2 + 0.03 * loss_similiar
loss_labelling.backward()
# compute target specific loss
loss_F_t = criterion(out_F_t, labels_tgt)
loss_F_t.backward()
# optimize
optimizer_F.step()
optimizer_F_1.step()
optimizer_F_2.step()
optimizer_F_t.step()
losses.append(loss_F_t.item())
# print step info
if ((step + 1) % cfg.log_step == 0):
print("K[{}/{}] Epoch [{}/{}] Step[{}/{}] Loss("
"labelling={:.5f} target={:.5f})".format(
k + 1,
cfg.num_epochs_k,
epoch + 1,
cfg.num_epochs_adapt,
step + 1,
len(merged_dataloader),
loss_labelling.item(), #.data[0],
loss_F_t.item(), #.data[0],
))
#!!print('end of loop')
if plot:
plt.clf()
plt.plot(losses)
plt.grid(1)
plt.title(
'Loss for domain adaptation, k = {}/{}, epoch = {}/{}'
.format(k, cfg.num_epochs_k, epoch,
cfg.num_epochs_adapt))
plt.waitforbuttonpress(0.0001)
# re-compute the number of selected taget data
num_target = (k + 2) * len(source_dataset) // 20
num_target = min(num_target, cfg.num_target_max)
print(">>> Set num of sampled target data: {}".format(num_target))
# re-generate pseudo labels
excerpt, pseudo_labels = generate_labels(F,
F_1,
F_2,
target_dataset,
num_target,
useWeightedSampling=True)
print(">>> Genrate pseudo labels [{}] numtarget = {}".format(
len(target_dataset_labelled), num_target))
print('sizes = {}, {}, excerpt = {}, \npseudo_labels = {}'.format(
len(excerpt), len(pseudo_labels), excerpt, pseudo_labels))
# get labelled target dataset
target_dataset_labelled = get_dummy(target_dataset,
excerpt,
pseudo_labels,
get_dataset=True)
# re-merge soruce data and target data
merged_dataset = ConcatDataset(
[source_dataset, target_dataset_labelled])
# save model
if ((k + 1) % cfg.save_step == 0):
save_model(F, "adapt-F-{}.pt".format(k + 1))
save_model(F_1, "adapt-F_1-{}.pt".format(k + 1))
save_model(F_2, "adapt-F_2-{}.pt".format(k + 1))
save_model(F_t, "adapt-F_t-{}.pt".format(k + 1))
# save final model
save_model(F, "adapt-F-final.pt")
save_model(F_1, "adapt-F_1-final.pt")
save_model(F_2, "adapt-F_2-final.pt")
save_model(F_t, "adapt-F_t-final.pt")
def pre_train(F, F_1, F_2, F_t, source_data, plot):
"""Pre-train models on source domain dataset."""
# set train state for Dropout and BN layers
F.train()
F_1.train()
F_2.train()
F_t.train()
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
if 0:
optimType = "Adam"
cfg.learning_rate = 1.0E-4
else:
optimType = "sgd"
cfg.learning_rate = 1.0E-3
optimizer_F = get_optimizer(F, optimType)
optimizer_F_1 = get_optimizer(F_1, optimType)
optimizer_F_2 = get_optimizer(F_2, optimType)
optimizer_F_t = get_optimizer(F_t, optimType)
losses = []
if plot:
plt.figure()
# start training
for epoch in range(cfg.num_epochs_pre):
if optimType == 'sgd':
adjustLearningRate(optimizer_F, cfg.learning_rate, epoch,
cfg.num_epochs_pre)
adjustLearningRate(optimizer_F_1, cfg.learning_rate, epoch,
cfg.num_epochs_pre)
adjustLearningRate(optimizer_F_2, cfg.learning_rate, epoch,
cfg.num_epochs_pre)
adjustLearningRate(optimizer_F_t, cfg.learning_rate, epoch,
cfg.num_epochs_pre)
for step, (images, labels) in enumerate(source_data):
# convert into torch.autograd.Variable
images = make_variable(images)
labels = make_variable(labels)
# zero-grad optimizer
optimizer_F.zero_grad()
optimizer_F_1.zero_grad()
optimizer_F_2.zero_grad()
optimizer_F_t.zero_grad()
# forward networks
out_F = F(images)
#!!
#out_F = torch.flatten(out_F,1)
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
out_F_t = F_t(out_F)
# compute loss
loss_similiar = calc_similiar_penalty(F_1, F_2)
loss_F_1 = criterion(out_F_1, labels)
loss_F_2 = criterion(out_F_2, labels)
loss_F_t = criterion(out_F_t, labels)
loss_F = loss_F_1 + loss_F_2 + loss_F_t + 0.03 * loss_similiar
loss_F.backward()
# optimize
optimizer_F.step()
optimizer_F_1.step()
optimizer_F_2.step()
optimizer_F_t.step()
losses.append(loss_F.item())
# print step info
if ((step + 1) % cfg.log_step == 0):
print("Epoch [{}/{}] Step[{}/{}] Loss("
"Total={:.5f} F_1={:.5f} F_2={:.5f} "
"F_t={:.5f} sim={:.5f})"
#!! "F_t={:.5f})"
.format(epoch + 1,
cfg.num_epochs_pre,
step + 1,
len(source_data),
loss_F.item(), #.data[0],
loss_F_1.item(), #.data[0],
loss_F_2.item(), #.data[0],
loss_F_t.item(), #.data[0],
loss_similiar.item(), #.data[0],
))
if plot:
plt.clf()
plt.plot(losses)
plt.grid(1)
plt.title('Loss for pre-training')
plt.waitforbuttonpress(0.0001)
# save model
if ((epoch + 1) % cfg.save_step == 0):
save_model(F, "pretrain-F-{}.pt".format(epoch + 1))
save_model(F_1, "pretrain-F_1-{}.pt".format(epoch + 1))
save_model(F_2, "pretrain-F_2-{}.pt".format(epoch + 1))
save_model(F_t, "pretrain-F_t-{}.pt".format(epoch + 1))
# save final model
save_model(F, "pretrain-F-final.pt")
save_model(F_1, "pretrain-F_1-final.pt")
save_model(F_2, "pretrain-F_2-final.pt")
save_model(F_t, "pretrain-F_t-final.pt")
def domain_adapt(F, F_1, F_2, F_t, source_dataset, target_dataset, excerpt,
pseudo_labels):
"""Perform Doamin Adaptation between source and target domains."""
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
optimizer_F = get_optimizer(F, "Adam")
optimizer_F_1 = get_optimizer(F_1, "Adam")
optimizer_F_2 = get_optimizer(F_2, "Adam")
optimizer_F_t = get_optimizer(F_t, "Adam")
# get labelled target dataset
target_dataset_labelled = get_dummy(target_dataset,
excerpt,
pseudo_labels,
get_dataset=True)
# merge soruce data and target data
merged_dataset = ConcatDataset([source_dataset, target_dataset_labelled])
# start training
for k in range(cfg.num_epochs_k):
# set train state for Dropout and BN layers
F.train()
F_1.train()
F_2.train()
F_t.train()
merged_dataloader = make_data_loader(merged_dataset)
target_dataloader_labelled = get_inf_iterator(
make_data_loader(target_dataset_labelled))
for epoch in range(cfg.num_epochs_adapt):
for step, (images, labels) in enumerate(merged_dataloader):
# sample from T_l
images_tgt, labels_tgt = next(target_dataloader_labelled)
# convert into torch.autograd.Variable
images = make_variable(images)
labels = make_variable(labels)
images_tgt = make_variable(images_tgt)
labels_tgt = make_variable(labels_tgt)
# zero-grad optimizer
optimizer_F.zero_grad()
optimizer_F_1.zero_grad()
optimizer_F_2.zero_grad()
optimizer_F_t.zero_grad()
# forward networks
out_F = F(images)
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
out_F_t = F_t(F(images_tgt))
# compute labelling loss
loss_similiar = calc_similiar_penalty(F_1, F_2)
loss_F_1 = criterion(out_F_1, labels)
loss_F_2 = criterion(out_F_2, labels)
loss_labelling = loss_F_1 + loss_F_2 + loss_similiar
loss_labelling.backward()
# compute target specific loss
loss_F_t = criterion(out_F_t, labels_tgt)
loss_F_t.backward()
# optimize
optimizer_F.step()
optimizer_F_1.step()
optimizer_F_2.step()
optimizer_F_t.step()
# print step info
if ((step + 1) % cfg.log_step == 0):
print("K[{}/{}] Epoch [{}/{}] Step[{}/{}] Loss("
"labelling={:.5f} target={:.5f})".format(
k + 1,
cfg.num_epochs_k,
epoch + 1,
cfg.num_epochs_adapt,
step + 1,
len(merged_dataloader),
loss_labelling.data[0],
loss_F_t.data[0],
))
# re-compute the number of selected taget data
num_target = (k + 2) * len(source_dataset) // 20
num_target = min(num_target, cfg.num_target_max)
print(">>> Set num of sampled target data: {}".format(num_target))
# re-generate pseudo labels
excerpt, pseudo_labels = genarate_labels(F, F_1, F_2, target_dataset,
num_target)
print(">>> Genrate pseudo labels [{}]".format(
len(target_dataset_labelled)))
# get labelled target dataset
target_dataset_labelled = get_dummy(target_dataset,
excerpt,
pseudo_labels,
get_dataset=True)
# re-merge soruce data and target data
merged_dataset = ConcatDataset(
[source_dataset, target_dataset_labelled])
# save model
if ((k + 1) % cfg.save_step == 0):
save_model(F, "adapt-F-{}.pt".format(k + 1))
save_model(F_1, "adapt-F_1-{}.pt".format(k + 1))
save_model(F_2, "adapt-F_2-{}.pt".format(k + 1))
save_model(F_t, "adapt-F_t-{}.pt".format(k + 1))
# save final model
save_model(F, "adapt-F-final.pt")
save_model(F_1, "adapt-F_1-final.pt")
save_model(F_2, "adapt-F_2-final.pt")
save_model(F_t, "adapt-F_t-final.pt")
def pre_train(F, F_1, F_2, F_t, source_data):
"""Pre-train models on source domain dataset."""
# set train state for Dropout and BN layers
F.train()
F_1.train()
F_2.train()
F_t.train()
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
optimizer_F = get_optimizer(F, "Adam")
optimizer_F_1 = get_optimizer(F_1, "Adam")
optimizer_F_2 = get_optimizer(F_2, "Adam")
optimizer_F_t = get_optimizer(F_t, "Adam")
# start training
for epoch in range(cfg.num_epochs_pre):
for step, (images, labels) in enumerate(source_data):
# convert into torch.autograd.Variable
images = make_variable(images)
labels = make_variable(labels)
# zero-grad optimizer
optimizer_F.zero_grad()
optimizer_F_1.zero_grad()
optimizer_F_2.zero_grad()
optimizer_F_t.zero_grad()
# forward networks
out_F = F(images)
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
out_F_t = F_t(out_F)
# compute loss
loss_similiar = calc_similiar_penalty(F_1, F_2)
loss_F_1 = criterion(out_F_1, labels)
loss_F_2 = criterion(out_F_2, labels)
loss_F_t = criterion(out_F_t, labels)
loss_F = loss_F_1 + loss_F_2 + loss_F_t + loss_similiar
loss_F.backward()
# optimize
optimizer_F.step()
optimizer_F_1.step()
optimizer_F_2.step()
optimizer_F_t.step()
# print step info
if ((step + 1) % cfg.log_step == 0):
print("Epoch [{}/{}] Step[{}/{}] Loss("
"Total={:.5f} F_1={:.5f} F_2={:.5f} "
"F_t={:.5f} sim={:.5f})".format(
epoch + 1,
cfg.num_epochs_pre,
step + 1,
len(source_data),
loss_F.data[0],
loss_F_1.data[0],
loss_F_2.data[0],
loss_F_t.data[0],
loss_similiar.data[0],
))
# save model
if ((epoch + 1) % cfg.save_step == 0):
save_model(F, "pretrain-F-{}.pt".format(epoch + 1))
save_model(F_1, "pretrain-F_1-{}.pt".format(epoch + 1))
save_model(F_2, "pretrain-F_2-{}.pt".format(epoch + 1))
save_model(F_t, "pretrain-F_t-{}.pt".format(epoch + 1))
# save final model
save_model(F, "pretrain-F-final.pt")
save_model(F_1, "pretrain-F_1-final.pt")
save_model(F_2, "pretrain-F_2-final.pt")
save_model(F_t, "pretrain-F_t-final.pt")
def train(classifier, generator, critic, src_data_loader, tgt_data_loader):
"""Train generator, classifier and critic jointly."""
####################
# 1. setup network #
####################
# set train state for Dropout and BN layers
classifier.train()
generator.train()
# set criterion for classifier and optimizers
criterion = nn.CrossEntropyLoss()
optimizer_c = get_optimizer(classifier, "Adam")
# zip source and target data pair
data_iter_src = get_inf_iterator(src_data_loader)
# counter
g_step = 0
####################
# 2. train network #
####################
for epoch in range(params.num_epochs):
###########################
# 2.1 train discriminator #
###########################
# requires to compute gradients for D
for p in critic.parameters():
p.requires_grad = True
# set steps for discriminator
if g_step < 25 or g_step % 500 == 0:
# this helps to start with the critic at optimum
# even in the first iterations.
critic_iters = 100
else:
critic_iters = params.d_steps
critic_iters = 0
# loop for optimizing discriminator
#for d_step in range(critic_iters):
# convert images into torch.Variable
images_src, labels_src = next(data_iter_src)
images_src = make_variable(images_src).cuda()
labels_src = make_variable(labels_src.squeeze_()).cuda()
# print(type(images_src))
########################
# 2.2 train classifier #
########################
# zero gradients for optimizer
optimizer_c.zero_grad()
# compute loss for critic
preds_c = classifier(generator(images_src))
c_loss = criterion(preds_c, labels_src)
# optimize source classifier
c_loss.backward()
optimizer_c.step()
g_step += 1
##################
# 2.4 print info #
##################
if ((epoch + 1) % 500 == 0):
# print("Epoch [{}/{}]:"
# "c_loss={:.5f}"
# "D(x)={:.5f}"
# .format(epoch + 1,
# params.num_epochs,
# c_loss.item(),
# ))
test(classifier, generator, src_data_loader, params.src_dataset)
if ((epoch + 1) % 500 == 0):
save_model(generator, "Mnist-generator-{}.pt".format(epoch + 1))
save_model(classifier, "Mnist-classifer{}.pt".format(epoch + 1))
def train_tgt(src_encoder, tgt_encoder, critic, src_data_loader,
tgt_data_loader, src_classifier, tgt_classifier,
tgt_data_loader_eval, generator, discriminator, Saver, logger):
"""Train encoder for target domain."""
torch.cuda.set_device(0)
####################
# 1. setup network #
####################
# set train state for Dropout and BN layers
src_classifier.eval()
src_encoder.eval()
tgt_encoder.train()
tgt_classifier.eval()
critic.train()
generator.train()
discriminator.train()
# setup criterion and optimizer
criterion = nn.CrossEntropyLoss()
criterionRec = torch.nn.MSELoss()
criterionGAN = loss.GANLoss()
optimizer_tgt = optim.Adam(tgt_encoder.parameters(),
lr=cfg.learning_rate_apt,
betas=(cfg.beta1, cfg.beta2))
optimizer_critic = optim.Adam(critic.parameters(),
lr=cfg.learning_rate_apt_D,
betas=(cfg.beta1, cfg.beta2))
optimizer_autoencoder_ge = optim.Adam(list(tgt_encoder.parameters()) +
list(generator.parameters()),
lr=cfg.learning_rate_apt,
betas=(cfg.beta1, cfg.beta2))
optimizer_autoencoder_ad = optim.Adam(discriminator.parameters(),
lr=cfg.learning_rate_apt_D,
betas=(cfg.beta1, cfg.beta2))
len_data_loader = min(len(src_data_loader), len(tgt_data_loader))
####################
# 2. train network #
####################
stepall = 0
for epoch in range(cfg.num_epochs):
# zip source and target data pair
data_zip = enumerate(zip(src_data_loader, tgt_data_loader))
for step, ((images_src, labels_src), (images_tgt,
labels_tgt)) in data_zip:
# make images variable
images_src = make_variable(images_src)
images_tgt = make_variable(images_tgt)
labels_src = make_variable(labels_src).long().squeeze()
labels_tgt = make_variable(labels_tgt).long().squeeze()
###########################
# train discriminator #
###########################
# zero gradients for optimizer
optimizer_critic.zero_grad()
# extract and concat features
feat_src = src_encoder(images_src)
feat_tgt = tgt_encoder(images_tgt)
feat_concat = torch.cat((feat_src, feat_tgt), 0)
# predict on discriminator
pred_concat = critic(feat_concat.detach())
# prepare real and fake label
label_src = make_variable(torch.ones(feat_src.size(0)).long())
label_tgt = make_variable(torch.zeros(feat_tgt.size(0)).long())
label_concat = torch.cat((label_src, label_tgt), 0)
# compute loss for critic
loss_critic = criterion(pred_concat, label_concat)
loss_critic.backward()
# optimize critic
optimizer_critic.step()
############################
# train target encoder #
############################
# zero gradients for optimizer
optimizer_tgt.zero_grad()
# predict on discriminator
pred_tgt = critic(feat_tgt)
# prepare fake labels
label_tgt = make_variable(torch.ones(feat_tgt.size(0)).long())
# compute loss for target encoder
loss_tgt = criterion(pred_tgt, label_tgt)
loss_tgt.backward()
# optimize target encoder
optimizer_tgt.step()
###########################
# train autoencoder #
###########################
# generator
optimizer_autoencoder_ge.zero_grad()
feat_src = src_encoder(images_src)
feat_src_reshape = (feat_src.unsqueeze(2)).unsqueeze(2)
reconst_src = generator(feat_src_reshape)
loss_ge_src = criterionRec(reconst_src, images_src)
feat_tgt = tgt_encoder(images_tgt)
feat_tgt_reshape = (feat_tgt.unsqueeze(2)).unsqueeze(2)
reconst_tgt = generator(feat_tgt_reshape)
# loss_ge_tgt = criterionRec(reconst_tgt, images_tgt)
reconst_src_G = discriminator(reconst_src)
reconst_tgt_G = discriminator(reconst_tgt)
loss_reconst_src_G = criterionGAN(reconst_src_G, True)
loss_reconst_tgt_G = criterionGAN(reconst_tgt_G, True)
loss_autoencoder_ge = cfg.para_const * loss_ge_src + loss_reconst_src_G + loss_reconst_tgt_G
loss_autoencoder_ge.backward()
optimizer_autoencoder_ge.step()
# discriminator
optimizer_autoencoder_ad.zero_grad()
reconst_src_D = discriminator(reconst_src.detach())
reconst_tgt_D = discriminator(reconst_tgt.detach())
loss_reconst_src_D = criterionGAN(reconst_src_D, True)
loss_reconst_tgt_D = criterionGAN(reconst_tgt_D, False)
loss_autoencoder_ad = (loss_reconst_src_D +
loss_reconst_tgt_D) * cfg.para_autoD
loss_autoencoder_ad.backward()
optimizer_autoencoder_ad.step()
#######################
# print step info #
#######################
acc_src = evaluate.evaluate_step(src_encoder, src_classifier,
images_src, labels_src)
acc_tgt = evaluate.evaluate_step(tgt_encoder, tgt_classifier,
images_tgt, labels_tgt)
infodic = OrderedDict([
('loss_reconst', loss_ge_src.data[0]),
('feat_d_loss', loss_critic.data[0]),
('feat_g_loss', loss_tgt.data[0]),
('auto_g_loss', loss_autoencoder_ge.data[0]),
('auto_d_loss', loss_autoencoder_ad.data[0]),
('acc_src', acc_src), ('acc_tgt', acc_tgt)
])
for tag, value in infodic.items():
logger.scalar_summary(tag, value, stepall)
if ((step + 1) % cfg.log_step == 0):
Saver.print_current_errors(epoch, (step + 1), infodic)
stepall += 1
# eval model on test set
evaluate.eval_func(tgt_encoder, tgt_classifier, tgt_data_loader_eval)
#############################
# 2.4 save model parameters #
#############################
if ((epoch + 1) % cfg.save_step == 0):
torch.save(
critic.state_dict(),
os.path.join(cfg.model_root, cfg.name,
"ADDA-critic-{}.pt".format(epoch + 1)))
torch.save(
tgt_encoder.state_dict(),
os.path.join(cfg.model_root, cfg.name,
"ADDA-target-encoder-{}.pt".format(epoch + 1)))
torch.save(
tgt_classifier.state_dict(),
os.path.join(
cfg.model_root, cfg.name,
"ADDA-target-tgt-classifier-{}.pt".format(epoch + 1)))
torch.save(
generator.state_dict(),
os.path.join(cfg.model_root, cfg.name,
"ADDA-generator-{}.pt".format(epoch + 1)))
torch.save(critic.state_dict(),
os.path.join(cfg.model_root, cfg.name, "ADDA-critic-final.pt"))
torch.save(
tgt_encoder.state_dict(),
os.path.join(cfg.model_root, cfg.name, "ADDA-target-encoder-final.pt"))
torch.save(
tgt_classifier.state_dict(),
os.path.join(cfg.model_root, cfg.name,
"ADDA-target-tgt-classifier-final.pt"))
torch.save(
generator.state_dict(),
os.path.join(cfg.model_root, cfg.name,
"ADDA-generator-final.pt".format(epoch + 1)))
return tgt_encoder, tgt_classifier
model = make_cuda(
inception_v3(pretrained=True, transform_input=True, extract_feat=True))
pca = PCAWrapper(n_components=cfg.n_components)
model.eval()
# data loader for frames in ingle video
# data_loader = get_dataloader(dataset="VideoFrame",
# path=cfg.video_file,
# num_frames=cfg.num_frames,
# batch_size=cfg.batch_size)
# data loader for frames decoded from several videos
data_loader = get_dataloader(dataset="FrameImage",
path=cfg.frame_root,
batch_size=cfg.batch_size)
# extract features by inception_v3
feats = None
for step, frames in enumerate(data_loader):
print("extracting feature [{}/{}]".format(step + 1, len(data_loader)))
feat = model(make_variable(frames))
feats = concat_feat(feats, feat.data.cpu())
# recude dimensions by PCA
X = feats.numpy()
pca.fit(X)
X_ = pca.transform(X)
print("reduce X {} to X_ {}".format(X.shape, X_.shape))
# sabe PCA params
pca.save_params(filepath=cfg.pca_model)
def generate_labels(F,
F_1,
F_2,
target_dataset,
num_target,
useWeightedSampling=False):
"""Genrate pseudo labels for target domain dataset."""
# set eval state for Dropout and BN layers
F.eval()
F_1.eval()
F_2.eval()
# get candidate samples
print("Num of sampled target data: {}".format(num_target))
# get sampled data loader
if useWeightedSampling:
classCounts, target_labels = atr.getClassCountsOfDataSet(
target_dataset)
assert np.all(classCounts > 0)
classWeights = 1.0 / classCounts.astype(float)
sampleWeights = classWeights[target_labels]
target_sampler = torch.utils.data.sampler.WeightedRandomSampler(
sampleWeights, num_target, replacement=True)
#data_loader = make_data_loader(target_dataset, sampler=target_sampler, shuffle=False)
else:
dummy = get_sampled_data_loader(target_dataset,
num_target,
shuffle=True)
target_sampler = dummy.sampler
# Use this sampler to select out the indices we want.
targetIndices = [z for z in target_sampler]
# Make a subset random dampler and data loader.
sampler = SubsetSampler(targetIndices)
data_loader = torch.utils.data.DataLoader(dataset=target_dataset,
batch_size=cfg.batch_size,
shuffle=False,
sampler=sampler)
# get output of F_1 and F_2 on sampled target dataset
out_F_1_total = None
out_F_2_total = None
gtTotal = None
for step, (images, gt) in enumerate(data_loader):
# convert into torch.autograd.Variable
images = make_variable(images)
# forward networks
out_F = F(images)
out_F_1 = F_1(out_F)
out_F_2 = F_2(out_F)
# concat outputs
if step == 0:
out_F_1_total = out_F_1.data.cpu()
out_F_2_total = out_F_2.data.cpu()
gtTotal = gt.data.cpu()
else:
out_F_1_total = torch.cat([out_F_1_total, out_F_1.data.cpu()], 0)
out_F_2_total = torch.cat([out_F_2_total, out_F_2.data.cpu()], 0)
gtTotal = torch.cat([gtTotal, gt.data.cpu()])
print('gt type = {}, val = {}'.format(type(gtTotal), gtTotal))
# guess pseudo labels
excerpt, pseudo_labels = \
guess_pseudo_labels(out_F_1_total, out_F_2_total)
#print('pl shape before = {}'.format(pseudo_labels.shape))
if 0:
# Using GT labels shows these are correct. The problem is that the labels from guess function are
# not correct.
pseudo_labels = torch.tensor([gtTotal[z] for z in excerpt])
#print('pl shape after = {}'.format(pseudo_labels.shape))
# Convert these indices into the subset into indices into target dataset.
excerpt = np.array([targetIndices[i] for i in excerpt])
assert len(excerpt) <= num_target
assert np.all(np.logical_and(excerpt >= 0, excerpt < len(target_dataset)))
print('Max excerpt = {}, but length of target data set = {}'.format(
excerpt.max(), len(target_dataset)))
return excerpt, pseudo_labels
