def run_epoch(self, epoch, training):
self.model.train(training)
if training:
description = '[Train]'
dataset = self.trainData
shuffle = True
else:
description = '[Valid]'
dataset = self.validData
shuffle = False
# dataloader for train and valid
dataloader = DataLoader(
dataset,
batch_size=self.batch_size,
shuffle=shuffle,
num_workers=8,
collate_fn=dataset.collate_fn,
)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description)
loss = 0
accuracy = Accuracy()
for i, (
x,
y,
) in trange: # (x,y) = 128*128
o_labels, batch_loss = self.run_iter(x, y)
if training:
self.opt.zero_grad() # reset gradient to 0
batch_loss.backward() # calculate gradient
self.opt.step() # update parameter by gradient
loss += batch_loss.item() # .item() to get python number in Tensor
accuracy.update(o_labels.cpu(), y)
trange.set_postfix(accuracy=accuracy.print_score(),
loss=loss / (i + 1))
if training:
self.history['train'].append({
'accuracy': accuracy.get_score(),
'loss': loss / len(trange)
})
self.scheduler.step()
else:
self.history['valid'].append({
'accuracy': accuracy.get_score(),
'loss': loss / len(trange)
})
if loss < self.min_loss:
self.save_best_model(epoch)
self.min_loss = loss
self.save_hist()
def run_epoch(self, epoch, training):
self.model.train(training)
if training:
description = 'Train'
dataset = self.trainData
shuffle = True
else:
description = 'Valid'
dataset = self.validData
shuffle = False
dataloader = DataLoader(dataset=dataset,
batch_size=self.batch_size,
shuffle=shuffle,
collate_fn=dataset.collate_fn,
num_workers=4)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description,
ascii=True)
f_loss = 0
l_loss = 0
accuracy = Accuracy()
for i, (x, missing, y) in trange:
o_labels, batch_f_loss, batch_l_loss = self.run_iter(x, missing, y)
batch_loss = batch_f_loss + batch_l_loss
if training:
self.opt.zero_grad()
batch_loss.backward()
self.opt.step()
f_loss += batch_f_loss.item()
l_loss += batch_l_loss.item()
accuracy.update(o_labels.cpu(), y)
trange.set_postfix(accuracy=accuracy.print_score(),
f_loss=f_loss / (i + 1),
l_loss=l_loss / (i + 1))
if training:
self.history['train'].append({
'accuracy': accuracy.get_score(),
'loss': f_loss / len(trange)
})
self.scheduler.step()
else:
self.history['valid'].append({
'accuracy': accuracy.get_score(),
'loss': f_loss / len(trange)
})
def run_epoch(self, epoch, source_dataloader, target_dataloader, lamb):
trange = tqdm(zip(source_dataloader, target_dataloader),
total=len(source_dataloader),
desc=f'[epoch {epoch}]')
total_D_loss, total_F_loss = 0.0, 0.0
acc = Accuracy()
for i, ((source_data, source_label), (target_data,
_)) in enumerate(trange):
source_data = source_data.to(self.device)
source_label = source_label.to(self.device)
target_data = target_data.to(self.device)
# =========== Preprocess =================
# mean/var of source and target datas are different, so we put them together for properly batch_norm
mixed_data = torch.cat([source_data, target_data], dim=0)
domain_label = torch.zeros(
[source_data.shape[0] + target_data.shape[0],
1]).to(self.device)
domain_label[:source_data.shape[
0]] = 1 # source data label=1, target data lebel=0
feature = self.feature_extractor(mixed_data)
# =========== Step 1 : Train Domain Classifier (fix feature extractor by feature.detach()) =================
domain_logits = self.domain_classifier(feature.detach())
loss = self.domain_criterion(domain_logits, domain_label)
total_D_loss += loss.item()
loss.backward()
self.optimizer_D.step()
# =========== Step 2: Train Feature Extractor and Label Predictor =================
class_logits = self.label_predictor(feature[:source_data.shape[0]])
domain_logits = self.domain_classifier(feature)
loss = self.class_criterion(
class_logits, source_label) - lamb * self.domain_criterion(
domain_logits, domain_label)
total_F_loss += loss.item()
loss.backward()
self.optimizer_F.step()
self.optimizer_C.step()
self.optimizer_D.zero_grad()
self.optimizer_F.zero_grad()
self.optimizer_C.zero_grad()
acc.update(class_logits, source_label)
trange.set_postfix(D_loss=total_D_loss / (i + 1),
F_loss=total_F_loss / (i + 1),
acc=acc.print_score())
self.history['d_loss'].append(total_D_loss / len(trange))
self.history['f_loss'].append(total_F_loss / len(trange))
self.history['acc'].append(acc.get_score())
self.save_hist()
self.save_model()
def run_epoch(self, epoch, dataset, training, desc=''):
self.model.train(training)
shuffle = training
dataloader = DataLoader(dataset, self.batch_size, shuffle=shuffle)
trange = tqdm(enumerate(dataloader), total=len(dataloader), desc=desc)
loss = 0
acc = Accuracy()
for i, (imgs, labels) in trange: # (b, 3, 128, 128), (b, 1)
labels = labels.view(-1) # (b,)
o_labels, batch_loss = self.run_iters(imgs, labels)
if training:
batch_loss /= self.accum_steps
batch_loss.backward()
if (i + 1) % self.accum_steps == 0:
self.opt.step()
self.opt.zero_grad()
batch_loss *= self.accum_steps
loss += batch_loss.item()
acc.update(o_labels.cpu(), labels)
trange.set_postfix(loss=loss / (i + 1), acc=acc.print_score())
if training:
self.history['train'].append({
'loss': loss / len(trange),
'acc': acc.get_score()
})
self.scheduler.step()
else:
self.history['valid'].append({
'loss': loss / len(trange),
'acc': acc.get_score()
})
if loss < self.best_score:
self.save_best()
self.save_hist()
def run_epoch(self, epoch, dataloader):
self.feature_extractor.train(True)
self.label_predictor.train(True)
trange = tqdm(dataloader,
total=len(dataloader),
desc=f'[epoch {epoch}]')
total_loss = 0
acc = Accuracy()
for i, (target_data, target_label) in enumerate(trange): # (b,1,32,32)
target_data = target_data.to(self.device)
target_label = target_label.view(-1).to(self.device) # (b)
feature = self.feature_extractor(target_data) # (b, 512)
class_logits = self.label_predictor(feature) # (b, 10)
loss = self.class_criterion(class_logits, target_label)
total_loss += loss.item()
loss.backward()
self.optimizer_F.step()
self.optimizer_C.step()
self.optimizer_F.zero_grad()
self.optimizer_C.zero_grad()
acc.update(class_logits, target_label)
trange.set_postfix(loss=total_loss / (i + 1),
acc=acc.print_score())
self.history['loss'].append(total_loss / len(trange))
self.history['acc'].append(acc.get_score())
self.save_hist()
self.save_model()
def run_epoch(self, epoch, training):
self.model.train(training)
self.generator.train(training)
self.discriminator.train(training)
if training:
description = 'Train'
dataset = self.trainData
shuffle = True
else:
description = 'Valid'
dataset = self.validData
shuffle = False
dataloader = DataLoader(dataset=dataset,
batch_size=self.batch_size,
shuffle=shuffle,
collate_fn=dataset.collate_fn,
num_workers=4)
trange = tqdm(enumerate(dataloader),
total=len(dataloader),
desc=description,
ascii=True)
g_loss = 0
d_loss = 0
loss = 0
accuracy = Accuracy()
for i, (features, real_missing, labels) in trange:
features = features.to(self.device) # (batch, 11)
real_missing = real_missing.to(self.device) # (batch, 3)
labels = labels.to(self.device) # (batch, 1)
batch_size = features.shape[0]
if training:
rand = torch.rand((batch_size, 11)).to(self.device) - 0.5
std = features.std(dim=1)
noise = rand * std.unsqueeze(1)
features += noise
# Adversarial ground truths
valid = torch.FloatTensor(batch_size, 1).fill_(1.0).to(
self.device) # (batch, 1)
fake = torch.FloatTensor(batch_size, 1).fill_(0.0).to(
self.device) # (batch, 1)
# ---------------------
# Train Discriminator
# ---------------------
if i % 10 < 5 or not training:
real_pred = self.discriminator(real_missing)
d_real_loss = self.criterion(real_pred, valid)
fake_missing = self.generator(features.detach())
fake_pred = self.discriminator(fake_missing)
d_fake_loss = self.criterion(fake_pred, fake)
batch_d_loss = (d_real_loss + d_fake_loss)
if training:
self.opt_D.zero_grad()
batch_d_loss.backward()
self.opt_D.step()
d_loss += batch_d_loss.item()
# -----------------
# Train Generator
# -----------------
if i % 10 >= 5 or not training:
gen_missing = self.generator(features.detach())
validity = self.discriminator(gen_missing)
batch_g_loss = self.criterion(validity, valid)
if training:
self.opt_G.zero_grad()
batch_g_loss.backward()
self.opt_G.step()
g_loss += batch_g_loss.item()
# ------------------
# Train Classifier
# ------------------
gen_missing = self.generator(features.detach())
all_features = torch.cat((features, gen_missing), dim=1)
o_labels = self.model(all_features)
batch_loss = self.criterion(o_labels, labels)
if training:
self.opt.zero_grad()
batch_loss.backward()
self.opt.step()
loss += batch_loss.item()
accuracy.update(o_labels, labels)
trange.set_postfix(accuracy=accuracy.print_score(),
g_loss=g_loss / (i + 1),
d_loss=d_loss / (i + 1),
loss=loss / (i + 1))
if training:
self.history['train'].append({
'accuracy': accuracy.get_score(),
'g_loss': g_loss / len(trange),
'd_loss': d_loss / len(trange),
'loss': loss / len(trange)
})
self.scheduler.step()
self.scheduler_G.step()
self.scheduler_D.step()
else:
self.history['valid'].append({
'accuracy': accuracy.get_score(),
'g_loss': g_loss / len(trange),
'd_loss': d_loss / len(trange),
'loss': loss / len(trange)
})
