def test(self, epoch):
self.data_loader.initialize(self.sess, is_train=False, is_val=False)
tt = tqdm(range(self.data_loader.num_iterations_test),
total=self.data_loader.num_iterations_test,
desc="Val-{}-".format(epoch))
loss_per_epoch = AverageMeter()
acc_per_epoch = AverageMeter()
for cur_it in tt:
loss, acc = self.sess.run([self.loss_node, self.acc_node],
feed_dict={self.is_training: False})
loss_per_epoch.update(loss)
acc_per_epoch.update(acc)
summaries_dict = {
'test/loss_per_epoch': loss_per_epoch.val,
'test/acc_per_epoch': acc_per_epoch.val
}
self.summarizer.summarize(
self.model.global_step_tensor.eval(self.sess), summaries_dict)
print("""Testing -> Val-{} loss:{:.4f} -- acc:{:.4f}""".format(
epoch, loss_per_epoch.val, acc_per_epoch.val))
tt.close()
def train_epoch(self, epoch=None):
self.data_loader.initialize(self.sess, is_train=True, is_val=False)
tt = tqdm(range(self.data_loader.num_iterations_train),
total=self.data_loader.num_iterations_train,
desc="epoch-{}-".format(epoch))
loss_per_epoch = AverageMeter()
acc_per_epoch = AverageMeter()
for cur_it in tt:
loss, acc = self.train_step()
loss_per_epoch.update(loss)
acc_per_epoch.update(acc)
self.sess.run(self.model.global_epoch_inc)
summaries_dict = {
'train/loss_per_epoch': loss_per_epoch.val,
'train/acc_per_epoch': acc_per_epoch.val
}
self.summarizer.summarize(
self.model.global_step_tensor.eval(self.sess), summaries_dict)
self.model.save(self.sess)
print("""Training -> Epoch-{} loss:{:.4f} -- acc:{:.4f}""".format(
epoch, loss_per_epoch.val, acc_per_epoch.val))
tt.close()
def validate(self):
# Eval mode
self.net.eval()
# Init Average Meters
epoch_s_acc = AverageMeter()
epoch_l_acc = AverageMeter()
tqdm_batch = tqdm(self.valid_loader, f'Epoch-{self.current_epoch}-')
with torch.no_grad():
for data in tqdm_batch:
# Prepare data
x = torch.tensor(data['x'], dtype=torch.float32, device=self.device)
y = torch.tensor(data['y'], dtype=torch.float32, device=self.device)
s_gt = torch.zeros_like(x)
s_gt[x != 0] = 1
# Forward pass
s_pred = self.net(y)
# Metrics
s_acc = eval_s_acc(s_pred, s_gt)
l_acc = eval_l_acc(s_pred, s_gt, self.cfg.NUM_Y)
batch_size = x.shape[0]
epoch_s_acc.update(s_acc.item(), batch_size)
epoch_l_acc.update(l_acc.item(), batch_size)
tqdm_batch.close()
print(f'Validate at epoch- {self.current_epoch} |'
f's_acc: {epoch_s_acc.val} - l_acc: {epoch_l_acc.val}')
def test(self, epoch):
# initialize dataset
self.data_loader.initialize(self.sess, mode='eval')
# initialize tqdm
tt = tqdm(range(self.data_loader.num_iterations_test), total=self.data_loader.num_iterations_test,
desc="Val-{}-".format(epoch))
loss_per_epoch = AverageMeter()
acc_per_epoch = AverageMeter()
# Iterate over batches
for cur_it in tt:
# One Train step on the current batch
loss, acc = self.sess.run([self.loss_node, self.acc_node],
feed_dict={self.is_training: False})
# update metrics returned from train_step func
loss_per_epoch.update(loss)
acc_per_epoch.update(acc)
# summarize
summaries_dict = {'eval/loss_per_epoch': loss_per_epoch.val,
'eval/acc_per_epoch': acc_per_epoch.val}
self.summarizer.summarize(self.model.global_step_tensor.eval(self.sess), summaries_dict)
print("""
Val-{} Eval loss:{:.4f} -- acc:{:.4f}
""".format(epoch, loss_per_epoch.val, acc_per_epoch.val))
tt.close()
def test(epoch):
progress_bar = tqdm(testloader)
net.eval()
acc_clean = AverageMeter()
acc_adv = AverageMeter()
for batch_idx, (images, labels) in enumerate(progress_bar):
images, labels = images.cuda(), labels.cuda()
with ctx_noparamgrad_and_eval(net):
images_adv = test_attacker.perturb(images, labels)
pred_clean = net(images).argmax(dim=1)
pred_adv = net(images_adv).argmax(dim=1)
acc_clean.update((pred_clean == labels).float().mean().item() * 100.0,
images.size(0))
acc_adv.update((pred_adv == labels).float().mean().item() * 100.0,
images.size(0))
progress_bar.set_description(
'Test Epoch: [{0}] '
'Clean Acc: {acc_clean.val:.3f} ({acc_clean.avg:.3f}) '
'Adv Acc: {acc_adv.val:.3f} ({acc_adv.avg:.3f}) '.format(
epoch, acc_clean=acc_clean, acc_adv=acc_adv))
logging.info(
f'Epoch: {epoch} | Clean: {acc_clean.avg:.2f} % | Adv: {acc_adv.avg:.2f} %'
)
def train_one_epoch(self):
self.model.train()
loss = AverageMeter()
acc = AverageMeter()
if self.verbose:
iterator = tqdm(self.train_loader)
else:
iterator = self.train_loader
for x, y in iterator:
x = x.to(self.device)
y = y.to(self.device)
output = self.model(x)
current_loss = self.loss(output, y)
self.optimizer.zero_grad()
current_loss.backward()
self.optimizer.step()
loss.update(current_loss.item())
output = output.detach().cpu().numpy()
y = y.cpu().numpy()
accuracy = get_accuracy(output, y)
acc.update(accuracy, y.shape[0])
# if self.mode == 'crossval':
s = ('Training epoch {} | loss: {} - accuracy: '
'{}'.format(self.cur_epoch, round(loss.val, 5), round(acc.val,
5)))
print_and_log(self.logger, s)
def train_one_epoch(self):
"""
One epoch of training
:return:
"""
self.loss_train_avg = AverageMeter()
self.mapk_train_avg = AverageMeter()
self.model.train()
count = 0
def logging(output, target, loss, batch_idx):
mapk3_metric = mapk3(output, target)
self.mapk_train_avg.update(mapk3_metric)
self.logger.info(
'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tMapk3: {:.6f}'
.format(self.current_epoch, batch_idx * len(data),
len(self.train_data_loader.dataset),
100. * batch_idx / len(self.train_data_loader),
loss.item(), mapk3_metric))
if self.summary_writer:
iteration = self.current_epoch * 1000 + int(
1000. * batch_idx / len(self.train_data_loader))
self.summary_writer.add_scalar("train_loss", loss.item(),
iteration)
self.summary_writer.add_scalar("train_mapk", mapk3_metric,
iteration)
self.summary_writer.add_scalar(
"lr", self.optimizer.param_groups[0]['lr'], iteration)
for batch_idx, (data, target) in enumerate(self.train_data_loader):
if count <= 0:
self.optimizer.step()
self.optimizer.zero_grad()
count = self.acum_batches
data, target = data.to(self.device), target.to(self.device)
output = self.model(data)
loss = self.loss(output, target)
## Loggin and gradient accum
if batch_idx % self.config.log_interval == 0:
logging(output, target, loss, batch_idx)
if self.acum_batches >= 2:
loss = loss / self.acum_batches
loss.backward()
self.loss_train_avg.update(loss.item())
self.current_iteration += 1
count = count - 1
if self.scheduler and "step_batch" in dir(self.scheduler):
self.scheduler.step_batch(self.current_iteration)
self.save_checkpoint()
def test(self, epoch):
self.data_loader.initialize(self.sess, mode='eval')
tt = tqdm(range(self.data_loader.num_iterations_test),
total=self.data_loader.num_iterations_test,
desc="Val-{}-".format(epoch))
loss_per_epoch = AverageMeter()
dice_per_epoch = AverageMeter()
iou_per_epoch = AverageMeter()
for cur_it in tt:
loss, dice, iou = self.sess.run(
[self.loss_node, self.dice_node, self.iou_node],
feed_dict={self.is_training: False})
loss_per_epoch.update(loss)
dice_per_epoch.update(dice)
iou_per_epoch.update(iou)
summaries_dict = {
'eval/loss_per_epoch': loss_per_epoch.val,
'eval/dice_per_epoch': dice_per_epoch.val,
'eval/iou_per_epoch': iou_per_epoch.val
}
self.summarizer.summarize(
self.model.global_step_tensor.eval(self.sess), summaries_dict)
print("""
Val-{} Eval loss:{:.4f} -- dice:{:.4f} -- iou:{:.4f}
""".format(epoch, loss_per_epoch.val, dice_per_epoch.val,
iou_per_epoch.val))
tt.close()
def validate(self):
self.model.eval()
loss = AverageMeter()
acc = AverageMeter()
for x, y in self.val_loader:
x = x.float()
x = x.to(self.device)
y = y.to(self.device)
output = self.model(x)
current_loss = self.loss(output, y)
loss.update(current_loss.item())
accuracy = get_accuracy(output, y)
acc.update(accuracy, y.shape[0])
s = ('Validating epoch {} | loss: {} - accuracy: '
'{}'.format(self.cur_epoch, round(loss.val, 5), round(acc.val,
5)))
# print_and_log(self.logger, s)
self.logger.info(s)
return acc.val, loss.val
def train_by_epoch(self):
tqdm_batch = tqdm(self.dataloader,
total=self.total_iter,
desc="epoch-{}".format(self.epoch))
avg_loss = AverageMeter()
for curr_it, (X, corner) in enumerate(tqdm_batch):
self.feature.train()
self.corner.train()
self.opt.zero_grad()
X = X.cuda(async=self.config.async_loading)
corner = corner.cuda(async=self.config.async_loading)
feat = self.feature(X)
pred_cor = self.corner(feat)
loss = self.loss(corner, pred_cor)
loss.backward()
self.opt.step()
avg_loss.update(loss)
if curr_it == 4:
self.record_image(X, pred_cor, corner)
tqdm_batch.close()
self.summary_writer.add_scalar('train/loss', avg_loss.val, self.epoch)
self.scheduler.step(avg_loss.val)
with torch.no_grad():
tqdm_batch = tqdm(
self.testloader,
total=(len(self.dataset_test) + self.config.batch_size - 1) //
self.config.batch_size,
desc="epoch-{}".format(self.epoch))
avg_loss = AverageMeter()
for curr_it, (X, corner) in enumerate(tqdm_batch):
self.feature.eval()
self.corner.eval()
X = X.cuda(async=self.config.async_loading)
corner = corner.cuda(async=self.config.async_loading)
feat = self.feature(X)
pred_cor = self.corner(feat)
loss = self.loss(corner, pred_cor)
avg_loss.update(loss)
if curr_it == 2:
self.record_image(X, pred_cor, corner, 'test')
tqdm_batch.close()
self.summary_writer.add_scalar('eval/loss', avg_loss.val,
self.epoch)
def train_one_epoch(self):
self.model.train()
loss = AverageMeter()
acc = AverageMeter()
for x, y in self.train_loader:
x = x.float()
x = x.to(self.device)
y = y.to(self.device)
output = self.model(x)
current_loss = self.loss(output, y)
self.optimizer.zero_grad()
current_loss.backward()
self.optimizer.step()
loss.update(current_loss.item())
accuracy = get_accuracy(output, y)
acc.update(accuracy, y.shape[0])
# if self.config.mode == 'crossval':
s = ('Training epoch {} | loss: {} - accuracy: '
'{}'.format(self.cur_epoch, round(loss.val, 5), round(acc.val,
5)))
# print_and_log(self.logger, s)
self.logger.info(s)
def test(self, state='test'):
# get loss and error_nodes
loss_node, = tf.get_collection('losses')
error_node, = tf.get_collection('error')
# initialize dataset
self.data_loader.initialize(self.sess, state=state)
# initialize tqdm
tt = tqdm(range(self.data_loader.num_iterations_test),
total=self.data_loader.num_iterations_test,
desc="test")
loss_per_epoch = AverageMeter()
error_per_epoch = AverageMeter()
# Iterate over batches
for _ in tt:
# One Train step on the current batch
loss, acc = self.sess.run([loss_node, error_node])
# update metrics returned from train_step func
loss_per_epoch.update(loss)
error_per_epoch.update(acc)
print("Test loss: {:.3E}, error: {:.2f}".format(
loss_per_epoch.val, error_per_epoch.val))
tt.close()
return loss_per_epoch.val, error_per_epoch.val
def train_one_epoch(self, specializing=False, cosine_decay=False):
"""
One epoch training function
:return:
"""
if specializing:
tqdm_batch = tqdm.tqdm(
self.sub_data_loader.binary_train_loader,
total=self.sub_data_loader.binary_train_iterations,
desc="Epoch-{}-".format(self.current_epoch))
else:
tqdm_batch = tqdm.tqdm(self.data_loader.train_loader,
total=self.data_loader.train_iterations,
desc="Epoch-{}-".format(self.current_epoch))
self.model.train()
epoch_loss = AverageMeter()
top1_acc = AverageMeter()
top5_acc = AverageMeter()
current_batch = 0
for i, (x, y) in enumerate(tqdm_batch):
if self.cuda:
x, y = x.cuda(non_blocking=self.config.async_loading), y.cuda(
non_blocking=self.config.async_loading)
self.optimizer.zero_grad()
if cosine_decay:
self.adjust_learning_rate(self.optimizer, self.current_epoch,
i, self.data_loader.train_iterations)
pred = self.model(x)
cur_loss = self.loss_fn(pred, y)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during training...')
cur_loss.backward()
self.optimizer.step()
if specializing:
top1 = cls_accuracy(pred.data, y.data)
top1_acc.update(top1[0].item(), x.size(0))
else:
top1, top5 = cls_accuracy(pred.data, y.data, topk=(1, 5))
top1_acc.update(top1.item(), x.size(0))
top5_acc.update(top5.item(), x.size(0))
epoch_loss.update(cur_loss.item())
self.current_iteration += 1
current_batch += 1
self.lr_list.append(self.optimizer.param_groups[0]['lr'])
tqdm_batch.close()
print("Training at epoch-" + str(self.current_epoch) + " | " +
"loss: " + str(epoch_loss.val) + "\tTop1 Acc: " +
str(top1_acc.val))
def test(self, epoch):
# initialize dataset
self.data_loader.initialize(self.sess, mode='test')
# initialize tqdm
tt = tqdm(range(self.data_loader.num_iterations_test),
total=self.data_loader.num_iterations_test,
desc="Val-{}-".format(epoch))
loss_per_epoch = AverageMeter()
kappa_per_epoch = AverageMeter()
# Iterate over batches
for cur_it in tt:
# One step on the current batch
loss, kappa = self.step()
# update metrics returned from step func
loss_per_epoch.update(loss)
kappa_per_epoch.update(kappa)
# summarize
summaries_dict = {
'test/loss_per_epoch': loss_per_epoch.val,
'test/kappa_per_epoch': kappa_per_epoch.val
}
self.summarizer.summarize(
self.model.global_step_tensor.eval(self.sess), summaries_dict)
print("""
Epoch-{} loss:{:.4f} -- kappa:{:.4f}
""".format(epoch, loss_per_epoch.val, kappa_per_epoch.val))
tt.close()
def test(model, loader, opt):
mean_miou = AverageMeter()
mean_loss = AverageMeter()
mean_acc = AverageMeter()
mean_time = AverageMeter()
with torch.no_grad():
for i, data in tqdm(enumerate(loader),
total=len(loader),
smoothing=0.9):
# if i % 50 ==0:
if not opt.multi_gpus:
data = data.to(opt.device)
target = data.y
batch_label2 = target.cpu().data.numpy()
inputs = torch.cat((data.pos.transpose(
2, 1).unsqueeze(3), data.x.transpose(2, 1).unsqueeze(3)), 1)
inputs = inputs[:, :self.num_channel, :, :]
gt = data.y.to(opt.device)
tic = time.perf_counter()
out = model(inputs)
toc = time.perf_counter()
print(f"Downloaded the tutorial in {toc - tic:0.4f} seconds")
# loss = criterion(out, gt)
# pdb.set_trace()
seg_pred = out.transpose(2, 1)
pred_val = seg_pred.contiguous().cpu().data.numpy()
seg_pred = seg_pred.contiguous().view(-1, opt.n_classes)
pred_val = np.argmax(pred_val, 2)
batch_label = target.view(-1, 1)[:, 0].cpu().data.numpy()
target = target.view(-1, 1)[:, 0]
pred_choice = seg_pred.cpu().data.max(1)[1].numpy()
correct = np.sum(pred_choice == batch_label)
current_seen_class = [0 for _ in range(opt.n_classes)]
current_correct_class = [0 for _ in range(opt.n_classes)]
current_iou_deno_class = [0 for _ in range(opt.n_classes)]
for l in range(opt.n_classes):
# pdb.set_trace()
current_seen_class[l] = np.sum((batch_label2 == l))
current_correct_class[l] = np.sum((pred_val == l)
& (batch_label2 == l))
current_iou_deno_class[l] = np.sum(
((pred_val == l) | (batch_label2 == l)))
m_iou = np.mean(
np.array(current_correct_class) /
(np.array(current_iou_deno_class, dtype=np.float) + 1e-6))
mean_time.update(toc - tic)
mean_miou.update(m_iou)
mean_acc.update(correct / float(opt.batch_size * NUM_POINT))
# pdb.set_trace()
test_time = mean_time.avg
test_mIoU = mean_miou.avg
test_macc = mean_acc.avg
print('Test point avg class IoU: %f' % (test_mIoU))
print('Test accuracy: %f' % (test_macc))
print('Test ave time(sec/frame): %f' % (test_time))
print('Test ave time(frame/sec): %f' % (1 / test_time))
def _reset_metrics(self):
self.batch_time = AverageMeter() # batch训练时间
self.data_time = AverageMeter() # 读取数据时间
self.total_loss = AverageMeter()
self.precision_top1, self.precision_top2 = AverageMeter(
), AverageMeter()
self.confusion_matrix = [[
0 for j in range(self.train_loader.dataset.num_classes)
] for i in range(self.train_loader.dataset.num_classes)]
def train_one_epoch(self):
# Set the model to be in training mode
self.net.train()
# Initialize average meters
epoch_loss = AverageMeter()
epoch_acc = AverageMeter()
epoch_iou = AverageMeter()
epoch_filtered_iou = AverageMeter()
tqdm_batch = tqdm(self.train_loader, f'Epoch-{self.current_epoch}-')
for x in tqdm_batch:
# prepare data
imgs = torch.tensor(x['img'],
dtype=torch.float,
device=self.device)
masks = torch.tensor(x['mask'],
dtype=torch.float,
device=self.device)
# model
pred, *_ = self.net(imgs)
# loss
cur_loss = self.loss(pred, masks)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during training...')
# optimizer
self.optimizer.zero_grad()
cur_loss.backward()
self.optimizer.step()
# metrics
pred_t = torch.sigmoid(pred) > 0.5
masks_t = masks > 0.5
cur_acc = torch.sum(pred_t == masks_t).item() / masks.numel()
cur_iou = iou_pytorch(pred_t, masks_t)
cur_filtered_iou = iou_pytorch(remove_small_mask_batch(pred_t),
masks_t)
batch_size = imgs.shape[0]
epoch_loss.update(cur_loss.item(), batch_size)
epoch_acc.update(cur_acc, batch_size)
epoch_iou.update(cur_iou.item(), batch_size)
epoch_filtered_iou.update(cur_filtered_iou.item(), batch_size)
tqdm_batch.close()
logging.info(
f'Training at epoch- {self.current_epoch} |'
f'loss: {epoch_loss.val:.5} - Acc: {epoch_acc.val:.5}'
f'- IOU: {epoch_iou.val:.5} - Filtered IOU: {epoch_filtered_iou.val:.5}'
)
def train_epoch(self, epoch=0):
losses = AverageMeter()
smooth_losses = AverageMeter()
image_losses = AverageMeter()
label_losses = AverageMeter()
len_train = len(self.train_loader)
pb = tqdm(self.train_loader)
self.model.train()
for i, (_, hsi) in enumerate(pb):
# sens for sensitivity and hsi for hyperspectral image
hsi = hsi.to(self.device)
sens = create_sensitivity('C').to(self.device)
# The reconstructed RGB in the range [0,1]
img_real = create_rgb(sens, hsi)
pred = self.model(img_real)
# Reconstruct RGB from sensitivity function and HSI
img_pred = create_rgb(pred, hsi)
smooth_loss = self.smooth_criterion(pred, pred.size(0))
image_loss = self.image_criterion(img_pred, img_real)
label_loss = self.label_criterion(pred, sens)
loss = self.calc_total_loss(image_loss, label_loss, smooth_loss)
losses.update(loss.item(), n=self.batch_size)
image_losses.update(image_loss.item(), n=self.batch_size)
label_losses.update(label_loss.item(), n=self.batch_size)
smooth_losses.update(smooth_loss.item(), n=self.batch_size)
# Compute gradients and do SGD step
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
desc = self.logger.make_desc(
i + 1, len_train,
('loss', losses, '.4f'),
('IL', image_losses, '.4f'),
('LL', label_losses, '.4f'),
('SL', smooth_losses, '.4f')
)
pb.set_description(desc)
self.logger.dump(desc)
# @zjw: tensorboard
self.logger.add_scalar('Estimator-Loss/train/total_losses', losses.val, epoch * len_train + i)
self.logger.add_scalar('Estimator-Loss/train/image_losses', image_losses.val, epoch * len_train + i)
self.logger.add_scalar('Estimator-Loss/train/label_losses', label_losses.val, epoch * len_train + i)
self.logger.add_scalar('Estimator-Loss/train/smooth_losses', smooth_losses.val, epoch * len_train + i)
self.logger.add_scalar('Estimator-Lr', self.optimizer.param_groups[0]['lr'], epoch * len_train + i)
def _train_epoch(self):
# Training mode
self.net.train()
# Init Average Meters
epoch_loss = AverageMeter()
epoch_acc = AverageMeter()
print("")
sp = StaticPrinter()
for step, inputs in enumerate(self.train_loader):
# Prepare data
for key, ipt in inputs.items():
inputs[key] = ipt.to(self.device)
# Forward pass
y_pred = self.net(inputs) # (batch, n_class)
# Compute loss
y_gt = inputs["label"] # (batch)
cur_loss = F.cross_entropy(y_pred, y_gt)
# Backprop and optimize
self.optimizer.zero_grad()
cur_loss.backward()
self.optimizer.step()
# Metrics
batch_size = y_gt.shape[0]
_, pred_idx = torch.max(y_pred, 1)
cur_acc = (pred_idx == y_gt).sum().item() / batch_size
""" # binary cross entropy 일 때의 예시.
y_pred_t = torch.sigmoid(y_pred) > 0.5
y_gt_t = y_gt > 0.5
cur_acc = (y_pred_t == y_gt_t).sum().item() / batch_size
"""
epoch_loss.update(cur_loss.item(), batch_size)
epoch_acc.update(cur_acc, batch_size)
# Print
sp.reset()
sp.print(
f"Epoch {self.epoch}| Training {step+1}/{len(self.train_loader)} | "
f"loss: {epoch_loss.val:.4f} - acc: {epoch_acc.val:.4f}"
)
logging.info(
f"Train at epoch {self.epoch} |"
f"loss: {epoch_loss.val:.4f} - acc: {epoch_acc.val:.4f}"
)
return epoch_acc.val
def _reset_metrics(self):
self.loss_sup = AverageMeter()
self.loss_unsup = AverageMeter()
self.loss_weakly = AverageMeter()
self.pair_wise = AverageMeter()
self.total_inter_l, self.total_union_l = 0, 0
self.total_correct_l, self.total_label_l = 0, 0
self.total_inter_ul, self.total_union_ul = 0, 0
self.total_correct_ul, self.total_label_ul = 0, 0
self.mIoU_l, self.mIoU_ul = 0, 0
self.pixel_acc_l, self.pixel_acc_ul = 0, 0
self.class_iou_l, self.class_iou_ul = {}, {}
def validate(self, specializing=False):
"""
One epoch validation
:return:
"""
if specializing:
tqdm_batch = tqdm.tqdm(
self.sub_data_loader.binary_valid_loader,
total=self.sub_data_loader.binary_valid_iterations,
desc="Epoch-{}-".format(self.current_epoch))
else:
tqdm_batch = tqdm.tqdm(self.data_loader.valid_loader,
total=self.data_loader.valid_iterations,
desc="Valiation at -{}-".format(
self.current_epoch))
self.model.eval()
epoch_loss = AverageMeter()
top1_acc = AverageMeter()
top5_acc = AverageMeter()
for x, y in tqdm_batch:
if self.cuda:
x, y = x.cuda(non_blocking=self.config.async_loading), y.cuda(
non_blocking=self.config.async_loading)
# model
pred = self.model(x)
# loss
cur_loss = self.loss_fn(pred, y)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during validation...')
if specializing:
top1 = cls_accuracy(pred.data, y.data)
top1_acc.update(top1[0].item(), x.size(0))
else:
top1, top5 = cls_accuracy(pred.data, y.data, topk=(1, 5))
top1_acc.update(top1.item(), x.size(0))
top5_acc.update(top5.item(), x.size(0))
epoch_loss.update(cur_loss.item())
self.logger.info("Validation results at epoch-" +
str(self.current_epoch) + " | " + "loss: " +
str(epoch_loss.avg) + "\tTop1 Acc: " +
str(top1_acc.val))
tqdm_batch.close()
return top1_acc.avg
def train_one_epoch(sess, ops, epoch, lr):
"""
One epoch training
"""
is_training = True
batch_time = AverageMeter()
loss_meter = AverageMeter()
weight_loss_meter = AverageMeter()
cls_loss_meter = AverageMeter()
acc_meter = AverageMeter()
sess.run(ops['train_init_op'])
feed_dict = {ops['is_training_pl']: is_training, ops['learning_rate']: lr}
batch_idx = 0
end = time.time()
while True:
try:
_, loss, classification_loss, weight_loss, \
tower_logits, tower_labels, = sess.run([ops['train_op'],
ops['loss'],
ops['classification_loss'],
ops['weight_loss'],
ops['tower_logits'],
ops['tower_labels']],
feed_dict=feed_dict)
for logits, labels in zip(tower_logits, tower_labels):
pred = np.argmax(logits, -1)
correct = np.mean(pred == labels)
acc_meter.update(correct, pred.shape[0])
# update meters
loss_meter.update(loss)
cls_loss_meter.update(classification_loss)
weight_loss_meter.update(weight_loss)
batch_time.update(time.time() - end)
end = time.time()
if (batch_idx + 1) % config.print_freq == 0:
logger.info(
f'Train: [{epoch}][{batch_idx}] '
f'T {batch_time.val:.3f} ({batch_time.avg:.3f}) '
f'acc {acc_meter.val:.3f} ({acc_meter.avg:.3f}) '
f'loss {loss_meter.val:.3f} ({loss_meter.avg:.3f}) '
f'cls loss {cls_loss_meter.val:.3f} ({cls_loss_meter.avg:.3f}) '
f'weight loss {weight_loss_meter.val:.3f} ({weight_loss_meter.avg:.3f})'
)
batch_idx += 1
except tf.errors.OutOfRangeError:
break
def train_one_epoch(sess, ops, epoch, lr):
"""
One epoch training
"""
is_training = True
batch_time = AverageMeter()
loss_meter = AverageMeter()
weight_loss_meter = AverageMeter()
seg_loss_meter = AverageMeter()
sess.run(ops['train_init_op'])
feed_dict = {ops['is_training_pl']: is_training, ops['learning_rate']: lr}
batch_idx = 0
end = time.time()
while True:
try:
_, loss, segment_loss, weight_loss = sess.run([
ops['train_op'], ops['loss'], ops['segment_loss'],
ops['weight_loss']
],
feed_dict=feed_dict)
loss_meter.update(loss)
seg_loss_meter.update(segment_loss)
weight_loss_meter.update(weight_loss)
batch_time.update(time.time() - end)
end = time.time()
if (batch_idx + 1) % config.print_freq == 0:
logger.info(
f'Train: [{epoch}][{batch_idx}] '
f'T {batch_time.val:.3f} ({batch_time.avg:.3f}) '
f'loss {loss_meter.val:.3f} ({loss_meter.avg:.3f}) '
f'seg loss {seg_loss_meter.val:.3f} ({seg_loss_meter.avg:.3f}) '
f'weight loss {weight_loss_meter.val:.3f} ({weight_loss_meter.avg:.3f})'
)
wandb.log({
"Train Loss": loss_meter.val,
"Train Avg Loss": loss_meter.avg,
"Train Seg Loss": seg_loss_meter.val,
"Train Avg Seg Loss": seg_loss_meter.avg,
"Train Weight Loss": weight_loss_meter.val,
"Train Avg Weight Loss": weight_loss_meter.avg
})
batch_idx += 1
except tf.errors.OutOfRangeError:
break
def validate_eval(val_loader, model, criterion, args, epoch=None, fnames=[]):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
model.eval()
end = time.time()
scores = np.zeros((len(val_loader.dataset), args.num_class))
labels = np.zeros((len(val_loader.dataset), ))
for i, (frames, objects, target) in enumerate(val_loader):
with torch.no_grad():
target = target.cuda(async=True)
frames = frames.cuda()
objects = objects.cuda()
output = model(frames, objects)
loss = criterion(output, target)
losses.update(loss.item(), target.size(0))
prec1 = accuracy(output.data, target)
top1.update(prec1[0], target.size(0))
batch_time.update(time.time() - end)
end = time.time()
# Record scores.
output_f = F.softmax(output, dim=1) # To [0, 1]
output_np = output_f.data.cpu().numpy()
labels_np = target.data.cpu().numpy()
b_ind = i * args.batch_size
e_ind = b_ind + min(args.batch_size, output_np.shape[0])
scores[b_ind:e_ind, :] = output_np
labels[b_ind:e_ind] = labels_np
print(
'Test [Epoch {0}/{1}]: '
'*Time {2:.2f}mins ({batch_time.avg:.2f}s) '
'*Loss {loss.avg:.4f} '
'*Prec@1 {top1.avg:.3f}'.format(epoch,
args.epoch,
batch_time.sum / 60,
batch_time=batch_time,
top1=top1,
loss=losses))
model.train()
res_scores = multi_scores(scores, labels,
['precision', 'recall', 'average_precision'])
return top1.avg, losses.avg, res_scores['precision'], res_scores[
'recall'], res_scores['average_precision']
def validate(self):
# set the model in eval mode
self.net.eval()
# Initialize average meters
epoch_loss = AverageMeter()
epoch_iou = AverageMeter()
epoch_filtered_iou = AverageMeter()
tqdm_batch = tqdm(self.valid_loader, f'Epoch-{self.current_epoch}-')
with torch.no_grad():
for x in tqdm_batch:
# prepare data
imgs = torch.tensor(x['img'],
dtype=torch.float,
device=self.device)
masks = torch.tensor(x['mask'],
dtype=torch.float,
device=self.device)
# model
pred, *_ = self.net(imgs)
# loss
cur_loss = self.loss(pred, masks)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during validation...')
# metrics
pred_t = torch.sigmoid(pred) > 0.5
masks_t = masks > 0.5
cur_iou = iou_pytorch(pred_t, masks_t)
cur_filtered_iou = iou_pytorch(remove_small_mask_batch(pred_t),
masks_t)
batch_size = imgs.shape[0]
epoch_loss.update(cur_loss.item(), batch_size)
epoch_iou.update(cur_iou.item(), batch_size)
epoch_filtered_iou.update(cur_filtered_iou.item(), batch_size)
tqdm_batch.close()
logging.info(f'Validation at epoch- {self.current_epoch} |'
f'loss: {epoch_loss.val:.5} - IOU: {epoch_iou.val:.5}'
f' - Filtered IOU: {epoch_filtered_iou.val:.5}')
return epoch_filtered_iou.val
def train_one_epoch(self, data_type = "None"):
"""
One epoch of training
:return:
"""
# Initialize tqdm
if data_type == "STRONG":
tqdm_batch = tqdm(self.data_loader.train_SE_loader, total=self.data_loader.train_SE_iterations,
desc="Epoch-{}-".format(self.current_epoch))
print( "Training with SEMEVAL data" )
else:
tqdm_batch = tqdm(self.data_loader.train_loader, total=self.data_loader.train_iterations,
desc="Epoch-{}-".format(self.current_epoch))
print( "Training with All week data and SEMEVAL data" )
self.model.train()
# Initialize your average meters
epoch_loss = AverageMeter()
epoch_acc = AverageMeter()
for itr, (x, y, _) in enumerate(tqdm_batch):
if self.cuda:
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
# model
pred = self.model(x.permute(1 ,0))
# loss
y = torch.max(y, 1)[1]
y = y.long()
cur_loss = self.loss(pred, y)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during training...')
# optimizer
self.optimizer.zero_grad()
cur_loss.backward()
self.optimizer.step()
epoch_loss.update(cur_loss.item())
batch_accuracy = accuracy(y, pred)
epoch_acc.update(batch_accuracy, x.size(0))
self.current_iteration += 1
self.summary_writer.add_scalar("epoch-training/loss", epoch_loss.val, self.current_iteration)
self.summary_writer.add_scalar("epoch_training/accuracy", epoch_acc.val, self.current_iteration)
tqdm_batch.close()
print( "Training Results at epoch-" + str(self.current_epoch) + " | " + "loss: " + str(epoch_loss.val) + " - acc-: " + str(epoch_acc.val ))
def train(model: nn.Module,
loader: DataLoader,
class_loss: nn.Module,
optimizer: Optimizer,
scheduler: _LRScheduler,
epoch: int,
callback: VisdomLogger,
freq: int,
ex: Experiment = None) -> None:
model.train()
device = next(model.parameters()).device
to_device = lambda x: x.to(device, non_blocking=True)
loader_length = len(loader)
train_losses = AverageMeter(device=device, length=loader_length)
train_accs = AverageMeter(device=device, length=loader_length)
pbar = tqdm(loader, ncols=80, desc='Training [{:03d}]'.format(epoch))
for i, (batch, labels, indices) in enumerate(pbar):
batch, labels, indices = map(to_device, (batch, labels, indices))
logits, features = model(batch)
loss = class_loss(logits, labels).mean()
acc = (logits.detach().argmax(1) == labels).float().mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
train_losses.append(loss)
train_accs.append(acc)
if callback is not None and not (i + 1) % freq:
step = epoch + i / loader_length
callback.scalar('xent',
step,
train_losses.last_avg,
title='Train Losses')
callback.scalar('train_acc',
step,
train_accs.last_avg,
title='Train Acc')
if ex is not None:
for i, (loss, acc) in enumerate(
zip(train_losses.values_list, train_accs.values_list)):
step = epoch + i / loader_length
ex.log_scalar('train.loss', loss, step=step)
ex.log_scalar('train.acc', acc, step=step)
def train_one_epoch(self):
"""
One epoch training function
:return:
"""
tqdm_batch = tqdm.tqdm(self.data_loader.train_loader,
total=self.data_loader.train_iterations,
desc="Epoch-{}-".format(self.current_epoch))
self.train()
epoch_loss = AverageMeter()
top1_acc = AverageMeter()
top5_acc = AverageMeter()
current_batch = 0
for i, (x, y) in enumerate(tqdm_batch):
if self.cuda:
x, y = x.cuda(non_blocking=self.config.async_loading), y.cuda(
non_blocking=self.config.async_loading)
self.optimizer.zero_grad()
# self.adjust_learning_rate(self.optimizer, self.current_epoch, i, self.data_loader.train_iterations)
pred = self(x)
cur_loss = self.loss_fn(pred, y)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during training...')
cur_loss.backward()
self.optimizer.step()
top1, top5 = cls_accuracy(pred.data, y.data, topk=(1, 5))
top1_acc.update(top1.item(), x.size(0))
top5_acc.update(top5.item(), x.size(0))
epoch_loss.update(cur_loss.item())
self.current_iteration += 1
current_batch += 1
tqdm_batch.close()
print("Training at epoch-" + str(self.current_epoch) + " | " +
"loss: " + str(epoch_loss.val) + "\tTop1 Acc: " +
str(top1_acc.val))
def train_one_epoch(self):
# Training mode
self.net.train()
# Init Average Meters
epoch_loss = AverageMeter()
epoch_s_acc = AverageMeter()
epoch_l_acc = AverageMeter()
tqdm_batch = tqdm(self.train_loader, f'Epoch-{self.current_epoch}-')
for data in tqdm_batch:
# Prepare data
x = torch.tensor(data['x'], dtype=torch.float32, device=self.device)
y = torch.tensor(data['y'], dtype=torch.float32, device=self.device)
s_gt = torch.zeros_like(x)
s_gt[x != 0] = 1
# Forward pass
s_pred = self.net(y)
# Compute loss
cur_loss = F.binary_cross_entropy_with_logits(s_pred, s_gt)
'''
cur_loss = F.log_softmax(s_pred, dim=1)
cur_loss = cur_loss * s_gt
cur_loss = - cur_loss.sum(dim=1)
cur_loss = cur_loss.mean()
'''
# Backprop and optimize
self.optim.zero_grad()
cur_loss.backward()
self.optim.step()
# Metrics
s_acc = eval_s_acc(s_pred, s_gt)
l_acc = eval_l_acc(s_pred, s_gt, self.cfg.NUM_Y)
batch_size = x.shape[0]
epoch_loss.update(cur_loss.item(), batch_size)
epoch_s_acc.update(s_acc.item(), batch_size)
epoch_l_acc.update(l_acc.item(), batch_size)
tqdm_batch.close()
print(f'Train at epoch- {self.current_epoch} |'
f'loss: {epoch_loss.val} - s_acc: {epoch_s_acc.val} - l_acc: {epoch_l_acc.val}')
def _predict_list(self, file_list, save_dir=None):
assert isinstance(file_list, list),\
"In 'list' mode the input must be a valid file_path list!"
consume_time = AverageMeter()
assert not len(file_list) == 0, "The input file list is empty!"
pb = tqdm(file_list) # processbar
for idx, path in enumerate(pb):
data = self._load_data(path)
name = os.path.basename(path)
data = self._np2tensor(data).unsqueeze(0).to(self.device)
path = os.path.join(save_dir, name) if save_dir else None
_, su_time = self.predict_base(model=self.model,
data=data,
path=path)
consume_time.update(su_time, n=1)
# logger
description = (
"[{}/{}] speed: {time.val:.4f}s({time.avg:.4f}s)".format(
idx + 1, len(file_list), time=consume_time))
pb.set_description(description)
self.logger.dump(description)