def criterion(inputs, target):
sigmoid = torch.sigmoid(inputs['out'])
sigmoid_aux = torch.sigmoid(inputs['aux'])
preds = (inputs['out'].data > 0).long()
loss = L.lovasz_softmax(sigmoid, target, classes=[1], ignore=128)
loss_aux = L.lovasz_softmax(sigmoid_aux, target, classes=[1], ignore=128)
iou = L.iou_binary(preds, target, ignore=128, per_image=True)
return loss + 0.5 * loss_aux, iou
def LovaszLossLayer(output, y):
# x: N,C,H,W
# cues: N,C+1,H,W
# y:N,1,H,W
# cues = cues.argmax(1)
# cues[cues == 2] = 10
# cues[cues == 7] = 10
# loss1=lovasz_softmax(x, cues, classes='present', per_image=False, ignore=10)
y = torch.squeeze(y, 1)
# y[y == 30] = 10
# y[y == 31] = 10
# y[y == 2] = 10
# y[y == 7] = 10
# #y[y == 9] = 10
y = y.cuda()
loss = lovasz_softmax(output,
y,
classes='present',
per_image=False,
ignore=10)
return loss
def val(epoch, args, criterion, val_loader, test_loader, filename=None):
fcn_model.eval()
total_ious = []
pixel_accs = []
iteration = 0
val_loss = 0
count = 0
for iter, (data, target) in tqdm.tqdm(
enumerate(val_loader), total=len(val_loader),
desc='Valid iteration=%d' % iteration, ncols=80,
leave=False):
if use_gpu:
inputs = Variable(data.cuda(gpu_used))
else:
inputs = Variable(data)
output = fcn_model(inputs)
if args.loss == "CE":
val_loss += criterion(output, target.cuda(gpu_used)).item()
else:
val_loss += L.lovasz_softmax(output, target.cuda(gpu_used), classes=[1]).item()
count = count + 1
output = output.data.cpu().numpy()
N, c, h, w = output.shape
pred = output.transpose(0, 2, 3, 1).reshape(-1, n_class).argmax(axis=1).reshape(N, h, w)
target = target.cpu().numpy().reshape(N, h, w)
for p, t in zip(pred, target):
total_ious.append(L.iou_binary(p, t))
pixel_accs.append(pixel_acc(p, t))
iteration += 1
val_loss /= count
pixel_accs = np.array(pixel_accs).mean()
print("epoch: {}, pix_acc: {}, IoU: {}, val_loss: {}".format(epoch, pixel_accs, np.mean(total_ious), val_loss))
if args.file == True:
csv_file = open(filename, "a")
csv_file.write(str(pixel_accs) + "," + str(np.mean(total_ious)) + "," + str(val_loss) + "\n")
csv_file.close()
early_stopping(np.mean(total_ious))#, model)
if early_stopping.early_stop:
print("Early stopping")
#test_set(test_loader)
test_set(test_loader, filename)
sys.exit()
def train(fcn_model,
args,
optimizer,
criterion,
scheduler,
train_loader,
val_loader,
filename=None,
filename1=None):
epochs = args.epochs
for epoch in range(epochs):
if args.scheduler == True:
scheduler.step()
ts = time.time()
for iter, (inputs, labels) in enumerate(train_loader):
optimizer.zero_grad()
if use_gpu:
#print("CUDA")
inputs = Variable(inputs.cuda(3))
labels = Variable(labels.cuda(3))
else:
#print("NO CUDA")
inputs, labels = Variable(inputs), Variable(labels)
N, c, h, w = inputs.shape
outputs = fcn_model(inputs)
# print("Pred: ", outputs)#.squeeze().shape)
# print("Labels: ", np.unique(labels.cpu().numpy(), return_counts=True))#[0, :, :, :].shape)
#outputs = outputs[:,1:,:]
if args.loss == "CE":
loss = criterion(outputs, labels)
else:
loss = L.lovasz_softmax(outputs, labels, classes=[1])
loss.backward()
optimizer.step()
print("Finish epoch {}, time elapsed {}".format(
epoch,
time.time() - ts))
print("epoch: {}, loss: {}".format(epoch, loss.data.item()))
# if args.file == True:
# csv_file = open(filename, "a")
# csv_file.write(str(epoch) + "," + str(loss.data.item()) + ",")
# csv_file.close()
val(fcn_model, epoch, args, criterion, val_loader, filename, filename1)
def validation(self, epoch):
self.model.eval()
self.evaluator.reset()
tbar = tqdm(self.val_loader, desc='\r')
test_loss = 0.0
for i, sample in enumerate(tbar):
image, target = sample['image'], sample['label']
if self.args.cuda:
image, target = image.cuda(), target.cuda()
with torch.no_grad():
#output = self.model(image)
output = nn.functional.softmax(self.model(image), dim=1)
#loss = self.criterion(output, target)
loss = L.lovasz_softmax(output, target)
test_loss += loss.item()
tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
pred = output.data.cpu().numpy()
target = target.cpu().numpy()
pred = np.argmax(pred, axis=1)
# Add batch sample into evaluator
self.evaluator.add_batch(target, pred)
# Fast test during the training
Acc = self.evaluator.Pixel_Accuracy()
Acc_class = self.evaluator.Pixel_Accuracy_Class()
mIoU = self.evaluator.Mean_Intersection_over_Union()
FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch)
self.writer.add_scalar('val/mIoU', mIoU, epoch)
self.writer.add_scalar('val/Acc', Acc, epoch)
self.writer.add_scalar('val/Acc_class', Acc_class, epoch)
self.writer.add_scalar('val/fwIoU', FWIoU, epoch)
print('Validation:')
print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU))
print('Loss: %.3f' % test_loss)
new_pred = mIoU
if new_pred > self.best_pred:
is_best = True
self.best_pred = new_pred
self.saver.save_checkpoint({
'epoch': epoch + 1,
'state_dict': self.model.module.state_dict(),
'optimizer': self.optimizer.state_dict(),
'best_pred': self.best_pred,
}, is_best)
def training(self, epoch):
train_loss = 0.0
self.model.train()
tbar = tqdm(self.train_loader)
num_img_tr = len(self.train_loader)
for i, sample in enumerate(tbar):
image, target = sample['image'], sample['label']
if self.args.cuda:
image, target = image.cuda(), target.cuda()
self.scheduler(self.optimizer, i, epoch, self.best_pred)
self.optimizer.zero_grad()
#output = self.model(image)
#loss = self.criterion(output, target)
output = nn.functional.softmax(self.model(image), dim=1)
loss = L.lovasz_softmax(output, target)
loss.backward()
self.optimizer.step()
train_loss += loss.item()
tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch)
# Show 10 * 3 inference results each epoch
if i % (num_img_tr // 10) == 0:
global_step = i + num_img_tr * epoch
self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step)
self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
print('Loss: %.3f' % train_loss)
if self.args.no_val:
# save checkpoint every epoch
is_best = False
self.saver.save_checkpoint({
'epoch': epoch + 1,
'state_dict': self.model.module.state_dict(),
'optimizer': self.optimizer.state_dict(),
'best_pred': self.best_pred,
}, is_best)
def train_model(model, optimizer, dataloaders, num_epochs=25):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_iou = 0.0
#class_weights=1/torch.Tensor(image_datasets['train'].get_label_ratio())
cel = CrossEntropyLoss2d(
) #weight=(class_weights*((1024**2)/class_weights.sum())).cuda())#my_model.Weighted_BCELoss(pos_weight=[0.0062,1])
dice = SoftDiceLoss()
customdice = CustomizedSoftDiceLoss()
focal = FocalLoss(gamma=2)
tod_loss = nn.BCEWithLogitsLoss()
tunnel_loss = nn.BCEWithLogitsLoss()
hdice = HardDiceLoss()
# Optimizerの第1引数には更新対象のfc層のパラメータのみ指定
#optimizer = optim.SGD(list(model.module.conv1.parameters())+list(model.module.fc.parameters()), lr=0.01, momentum=0.9)
scheduler = lr_scheduler.CosineAnnealingLR(optimizer,
num_epochs,
eta_min=0.00001,
last_epoch=-1)
#scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[100], gamma=0.1)
loss_dict = {
x: np.array([np.nan] * num_epochs)
for x in ['train', "validation"]
}
iou_dict = {
x: np.array([np.nan] * num_epochs * 4).reshape(num_epochs, 4)
for x in ['train', "validation"]
}
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
# 各エポックで訓練+バリデーションを実行
for phase in ['train', "validation"]:
if phase == 'train':
scheduler.step()
model.train(True) # training mode
else:
model.train(False) # evaluate mode
running_loss = 0.0
running_iou = np.zeros(4)
n_time = time.time()
for step, data in enumerate(tqdm(
dataloaders[phase])): #tqdm(dataloaders[phase]):
inputs, labels = data
if use_gpu:
inputs = torch.Tensor(inputs).to(device) #.unsqueeze_(1)
labels = torch.Tensor(labels).to(device)
#tod_labels=torch.Tensor(tod_labels.float()).to(device)
else:
inputs = torch.Tensor(inputs) #.unsqueeze_(1)
labels = torch.Tensor(labels) #.float()
batch_size, n_input_channel, img_height, img_width = tuple(
inputs.shape)
optimizer.zero_grad()
if phase == 'train':
outputs = model(inputs)
else:
with torch.no_grad():
outputs = model(inputs)
# label_weight_sum=labels.sum(dim=(0,2,3))
# label_weight_sum[label_weight_sum==0]=1
# class_weights=1/label_weight_sum
# if ip==3:#128.6
# loss = cel(outputs, labels.argmax(1))+customdice(outputs, labels)#AZ.log()#+0.1*tod_loss(tod_outputs[:,0],tod_labels[:,0])+0.1*tunnel_loss(tod_outputs[:,1],tod_labels[:,1])#cel(outputs, labels.argmax(1))+dice(outputs,labels).log()#weight=class_weights)(outputs, labels.argmax(1))
# elif ip==5:#128.8
# loss = cel(outputs, labels.argmax(1))+L.lovasz_softmax(F.softmax(outputs), labels.argmax(1), per_image=True)
# elif ip==2:#128.11
# loss=cel(outputs, labels.argmax(1))+dice(outputs, labels, per_image=True)#hdice(F.softmax(outputs),labels)
# elif ip==4:#128.7
# loss=cel(outputs, labels.argmax(1))+dice(outputs, labels)
# elif ip==7:#128.10
if epoch < 100:
loss = cel(
outputs, labels.argmax(1)
) #+0.75*L.lovasz_softmax(F.softmax(outputs), labels.argmax(1), per_image=True)
else:
loss = 0.25 * cel(outputs, labels.argmax(
1)) + 0.75 * L.lovasz_softmax(F.softmax(outputs),
labels.argmax(1),
per_image=True)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() #* batch_size
running_iou += iou(outputs, labels,
average=False) #.item()#*batch_size
torch.cuda.empty_cache()
# サンプル数で割って平均を求める
epoch_loss = running_loss / (
step + 1) #dataset_sizes[phase]#dataset_sizes[phase]
epoch_iou = running_iou / (
step + 1) #dataset_sizes[phase]#dataset_sizes[phase]
print('{} Loss: {:.4f} IOU: {:.4f}'.format(phase, epoch_loss,
epoch_iou.mean()))
#print('{} Loss: {:.4f} '.format(phase, epoch_loss))
loss_dict[phase][epoch] = epoch_loss
iou_dict[phase][epoch] = epoch_iou
#visdom
if phase == "validation":
output_img = np.zeros((3, img_height, img_width))
label_img = np.zeros((3, img_height, img_width))
output_argmax = outputs[0].argmax(0) #(height,width)
for idx, cla in enumerate(
image_datasets["train"].category_list):
if idx == 4:
break
#for y in range(img_height):
#for x in range(img_width):
#print(cla,labels[0,idx].cpu().data.numpy().sum(),np.array(image_datasets["train"].category_list[cla]).reshape((3,1,1)))
output_img += ((output_argmax
== idx).float().cpu().data.numpy().reshape(
(1, img_height, img_width)) *
np.array(image_datasets["train"].
category_list[cla]).reshape(
(3, 1, 1)))
# if cla=="car":
# print(label_img)
# print(labels[0,idx].sum().cpu().data.numpy())
# print(image_datasets["train"].category_list[cla])
label_img += (labels[0, idx].cpu().data.numpy().reshape(
(1, img_height, img_width)) *
np.array(image_datasets["train"].
category_list[cla]).reshape(
(3, 1, 1)))
#if idx==3:
#break
#print(output_img.shape)
win_output = viz.image(output_img / 255,
win="output",
opts=dict(title='output'))
win_label = viz.image(label_img / 255,
win="label",
opts=dict(title='label'))
win_input = viz.image(inputs[0].cpu().data.numpy(),
win="input",
opts=dict(title='input'))
if epoch > 0:
viz.line(X=np.arange(epoch + 1),
Y=loss_dict["train"][:epoch + 1],
update="replace",
win="loss",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1].mean(1),
update="replace",
win="iou",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 0],
update="replace",
win="car",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 1],
update="replace",
win="signal",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 2],
update="replace",
win="pedestrian",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 3],
update="replace",
win="lane",
name="train")
viz.line(X=np.arange(epoch + 1),
Y=loss_dict["validation"][:epoch + 1],
update="replace",
win="loss",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1].mean(1),
update="replace",
win="iou",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 0],
update="replace",
win="car",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 1],
update="replace",
win="signal",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 2],
update="replace",
win="pedestrian",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 3],
update="replace",
win="lane",
name="validation")
else:
win_loss = viz.line(X=np.arange(epoch + 1),
Y=loss_dict["train"][:epoch + 1],
win="loss",
name="train",
opts=dict(title='loss'))
win_iou = viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1].mean(1),
win="iou",
name="train",
opts=dict(title='iou'))
win_car = viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 0],
win="car",
name="train",
opts=dict(title='car'))
win_sig = viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 1],
win="signal",
name="train",
opts=dict(title='sig'))
win_ped = viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 2],
win="pedestrian",
name="train",
opts=dict(title='ped'))
win_lan = viz.line(X=np.arange(epoch + 1),
Y=iou_dict["train"][:epoch + 1, 3],
win="lane",
name="train",
opts=dict(title='lan'))
viz.line(X=np.arange(epoch + 1),
Y=loss_dict["validation"][:epoch + 1],
win="loss",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1].mean(1),
win="iou",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 0],
win="car",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 1],
win="signal",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 2],
win="pedestrian",
name="validation")
viz.line(X=np.arange(epoch + 1),
Y=iou_dict["validation"][:epoch + 1, 3],
win="lane",
name="validation")
# deep copy the model
# 精度が改善したらモデルを保存する
if phase == "validation" and epoch_iou.mean() > 0.65:
#print("save weights...",end="")
best_iou = epoch_iou.mean()
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(
model.state_dict(),
os.path.join(
save_path,
"{}_{:.4f}_{:.4f}.pth".format(epoch, epoch_loss,
epoch_iou.mean())))
#print("complete")
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val iou: {:.4f}'.format(best_iou))
# load best model weights
model.load_state_dict(best_model_wts)
return model
def keras_lovasz_softmax(labels, probas):
max_labels = tf.argmax(labels, axis=-1)
return lovasz_softmax(probas,
max_labels,
classes="present",
per_image=True)
val_fp = np.zeros(n_classes)
val_fn = np.zeros(n_classes)
net.train(True)
for data1 in tqdm.tqdm(trainDataLoader):
imgs, mask = data1
# print(imgs.shape)
if use_gpu:
inputs = imgs.cuda()
labels = mask.cuda()
cpmap = net(Variable(inputs))
cpmapD = F.softmax(cpmap, dim=1)
LGce = criterionS(cpmap, labels.long())
L_lov = L.lovasz_softmax(F.softmax(cpmap, dim=1), labels)
LGseg = LGce + L_lov
optimizerS.zero_grad()
LGseg.backward()
optimizerS.step()
trainRunningLoss += LGseg.item()
train_cf = dice_coefficient(cpmapD, labels)
train_tp += train_cf[0]
train_fp += train_cf[1]
train_fn += train_cf[2]
trainBatches += 1
super().__init__()
self.losses = [
nn.CrossEntropyLoss(),
kornia.losses.DiceLoss(),
kornia.losses.FocalLoss(alpha=.5, gamma=2, reduction='mean')
]
def __call__(self, pred, true):
return sum(loss(pred, true) for loss in self.losses)
sys.path.append('../external_packages/LovaszSoftmax/pytorch/')
sys.path.append('../../external_packages/LovaszSoftmax/pytorch/')
from lovasz_losses import lovasz_hinge, lovasz_softmax
LovaszLoss = (lambda pred, true: lovasz_softmax(
F.softmax(pred, dim=1), true, per_image=True))
# def read_dir(dirpath, ext="PNG"):
# df = pd.DataFrame(dict(img_path=sorted(glob(path.join(dirpath, f"*.{ext}")))))
# df["img"] = (df.img_path
# .apply(imread))
# return df
# def list_dir(dirpath, name, ext="png"):
# df = pd.DataFrame({name: glob(path.join(dirpath, f"*.{ext}"))})
# df["ID"] = (
# df[name]
# .apply(lambda filepath:
# path.splitext(path.split(filepath)[1])[0]
# )
def build_model(main_prog, start_prog, phase=ModelPhase.TRAIN):
if not ModelPhase.is_valid_phase(phase):
raise ValueError("ModelPhase {} is not valid!".format(phase))
if ModelPhase.is_train(phase):
width = cfg.TRAIN_CROP_SIZE[0]
height = cfg.TRAIN_CROP_SIZE[1]
else:
width = cfg.EVAL_CROP_SIZE[0]
height = cfg.EVAL_CROP_SIZE[1]
image_shape = [-1, cfg.DATASET.DATA_DIM, height, width]
grt_shape = [-1, 1, height, width]
class_num = cfg.DATASET.NUM_CLASSES
with fluid.program_guard(main_prog, start_prog):
with fluid.unique_name.guard():
# 在导出模型的时候，增加图像标准化预处理,减小预测部署时图像的处理流程
# 预测部署时只须对输入图像增加batch_size维度即可
if ModelPhase.is_predict(phase):
if cfg.SLIM.PREPROCESS:
image = fluid.data(
name='image', shape=image_shape, dtype='float32')
else:
origin_image = fluid.data(
name='image',
shape=[-1, -1, -1, cfg.DATASET.DATA_DIM],
dtype='float32')
image, valid_shape, origin_shape = export_preprocess(
origin_image)
else:
image = fluid.data(
name='image', shape=image_shape, dtype='float32')
label = fluid.data(name='label', shape=grt_shape, dtype='int32')
mask = fluid.data(name='mask', shape=grt_shape, dtype='int32')
# use DataLoader when doing traning and evaluation
if ModelPhase.is_train(phase) or ModelPhase.is_eval(phase):
data_loader = fluid.io.DataLoader.from_generator(
feed_list=[image, label, mask],
capacity=cfg.DATALOADER.BUF_SIZE,
iterable=False,
use_double_buffer=True)
loss_type = cfg.SOLVER.LOSS
if not isinstance(loss_type, list):
loss_type = list(loss_type)
# lovasz_hinge_loss或dice_loss或bce_loss只适用两类分割中
if class_num > 2 and (("lovasz_hinge_loss" in loss_type) or
("dice_loss" in loss_type) or
("bce_loss" in loss_type)):
raise Exception(
"lovasz hinge loss, dice loss and bce loss are only applicable to binary classfication."
)
# 在两类分割情况下，当loss函数选择lovasz_hinge_loss或dice_loss或bce_loss的时候，最后logit输出通道数设置为1
if ("dice_loss" in loss_type) or ("bce_loss" in loss_type) or (
"lovasz_hinge_loss" in loss_type):
class_num = 1
if ("softmax_loss" in loss_type) or (
"lovasz_softmax_loss" in loss_type):
raise Exception(
"softmax loss or lovasz softmax loss can not combine with bce loss or dice loss or lovasz hinge loss."
)
logits = seg_model(image, class_num)
# 根据选择的loss函数计算相应的损失函数
if ModelPhase.is_train(phase) or ModelPhase.is_eval(phase):
loss_valid = False
avg_loss_list = []
valid_loss = []
if "softmax_loss" in loss_type:
weight = cfg.SOLVER.CROSS_ENTROPY_WEIGHT
avg_loss_list.append(
multi_softmax_with_loss(logits, label, mask, class_num,
weight))
loss_valid = True
valid_loss.append("softmax_loss")
if "dice_loss" in loss_type:
avg_loss_list.append(multi_dice_loss(logits, label, mask))
loss_valid = True
valid_loss.append("dice_loss")
if "bce_loss" in loss_type:
avg_loss_list.append(multi_bce_loss(logits, label, mask))
loss_valid = True
valid_loss.append("bce_loss")
if "lovasz_hinge_loss" in loss_type:
avg_loss_list.append(
lovasz_hinge(logits, label, ignore=mask))
loss_valid = True
valid_loss.append("lovasz_hinge_loss")
if "lovasz_softmax_loss" in loss_type:
probas = fluid.layers.softmax(logits, axis=1)
avg_loss_list.append(
lovasz_softmax(probas, label, ignore=mask))
loss_valid = True
valid_loss.append("lovasz_softmax_loss")
if not loss_valid:
raise Exception(
"SOLVER.LOSS: {} is set wrong. it should "
"include one of (softmax_loss, bce_loss, dice_loss, lovasz_hinge_loss, lovasz_softmax_loss) at least"
" example: ['softmax_loss'], ['dice_loss'], ['bce_loss', 'dice_loss'], ['lovasz_hinge_loss','bce_loss'], ['lovasz_softmax_loss','softmax_loss']"
.format(cfg.SOLVER.LOSS))
invalid_loss = [x for x in loss_type if x not in valid_loss]
if len(invalid_loss) > 0:
print(
"Warning: the loss {} you set is invalid. it will not be included in loss computed."
.format(invalid_loss))
avg_loss = 0
for i in range(0, len(avg_loss_list)):
loss_name = valid_loss[i].upper()
loss_weight = eval('cfg.SOLVER.LOSS_WEIGHT.' + loss_name)
avg_loss += loss_weight * avg_loss_list[i]
#get pred result in original size
if isinstance(logits, tuple):
logit = logits[0]
else:
logit = logits
if logit.shape[2:] != label.shape[2:]:
logit = fluid.layers.resize_bilinear(logit, label.shape[2:])
# return image input and logit output for inference graph prune
if ModelPhase.is_predict(phase):
# 两类分割中，使用lovasz_hinge_loss或dice_loss或bce_loss返回的logit为单通道，进行到两通道的变换
if class_num == 1:
logit = sigmoid_to_softmax(logit)
else:
logit = softmax(logit)
# 获取有效部分
if cfg.SLIM.PREPROCESS:
return image, logit
else:
logit = fluid.layers.slice(
logit, axes=[2, 3], starts=[0, 0], ends=valid_shape)
logit = fluid.layers.resize_bilinear(
logit,
out_shape=origin_shape,
align_corners=False,
align_mode=0)
logit = fluid.layers.argmax(logit, axis=1)
return origin_image, logit
if class_num == 1:
out = sigmoid_to_softmax(logit)
out = fluid.layers.transpose(out, [0, 2, 3, 1])
else:
out = fluid.layers.transpose(logit, [0, 2, 3, 1])
pred = fluid.layers.argmax(out, axis=3)
pred = fluid.layers.unsqueeze(pred, axes=[3])
if ModelPhase.is_visual(phase):
if class_num == 1:
logit = sigmoid_to_softmax(logit)
else:
logit = softmax(logit)
return pred, logit
if ModelPhase.is_eval(phase):
return data_loader, avg_loss, pred, label, mask
if ModelPhase.is_train(phase):
optimizer = solver.Solver(main_prog, start_prog)
decayed_lr = optimizer.optimise(avg_loss)
return data_loader, avg_loss, decayed_lr, pred, label, mask
def forward(self, output, target):
logits = F.softmax(output, dim=1)
loss = lovasz_softmax(logits, target, ignore=self.ignore_index)
return loss
def forward(self, outputs, targets):
outputs = torch.sigmoid(outputs)
outputs = outputs.squeeze(1)
targets = targets.squeeze(1)
loss = L.lovasz_softmax(outputs,targets, classes=[1], ignore=255)
return loss
def val(fcn_model,
epoch,
args,
criterion,
val_loader,
filename=None,
filename1=None):
fcn_model.eval()
total_ious = []
pixel_accs = []
# pixel_background = []
# pixel_building = []
iteration = 0
val_loss = 0
count = 0
for iter, (data, target) in enumerate(val_loader):
if use_gpu:
#print("CUDA")
inputs = Variable(data.cuda(3))
else:
#print("NO CUDA")
inputs = Variable(data)
output = fcn_model(inputs)
if args.loss == "CE":
val_loss += criterion(output, target.cuda(3)).item()
else:
val_loss += L.lovasz_softmax(output, target.cuda(3),
classes=[1]).item()
count = count + 1
output = output.data.cpu().numpy()
N, c, h, w = output.shape
pred = output.transpose(0, 2, 3,
1).reshape(-1, n_class).argmax(axis=1).reshape(
N, h, w)
#pred = output.transpose(0, 2, 3, 1).reshape(-1, 1).argmax(axis=1).reshape(N, h, w)
target = target.cpu().numpy().reshape(N, h, w)
# pixel_building.append(np.unique(target, return_counts=True)[1].item(1))
# pixel_background.append(np.unique(target, return_counts=True)[1].item(0))
# N, c, h, w = inputs.shape
# input_print = inputs.cpu().numpy().reshape(c, h, w).transpose(1,2,0)
if (iter + 1) % 700 == 0:
now = datetime.datetime.now()
#if args.file == True:
img_name = "predictions/" + str(now.day) + "_" + str(
now.hour) + "_" + str(now.minute) + "-" + str(
epoch) + "-" + str(iter) + "-prediction.jpg"
targ_name = "predictions/" + str(now.day) + "_" + str(
now.hour) + "_" + str(now.minute) + "-" + str(
epoch) + "-" + str(iter) + "-truth.jpg"
#inp_name = "predictions/" + str(now.day) + "_" + str(now.hour) + "_" + str(now.minute) + "-" + str(epoch) + "-" + str(iter) + "-raw.jpg"
scipy.misc.imsave(img_name, pred.squeeze())
scipy.misc.imsave(targ_name, target.squeeze())
#scipy.misc.imsave(inp_name, input_print)
for p, t in zip(pred, target):
# total_ious.append(L.iou_binary(p, t))
total_ious.append(L.iou_binary(p, t))
pixel_accs.append(pixel_acc(p, t))
iteration += 1
val_loss /= count
pixel_accs = np.array(pixel_accs).mean()
print("epoch: {}, pix_acc: {}, IoU: {}, val_loss: {}".format(
epoch, pixel_accs, np.mean(total_ious), val_loss))
# if args.file == True:
# csv_file = open(filename, "a")
# csv_file.write(str(pixel_accs) + "," + str(np.mean(total_ious)) + "," + str(val_loss) + "\n")
# csv_file.close()
early_stopping(np.mean(total_ious)) #, model)
if early_stopping.early_stop:
print("Early stopping")
end_time = time.time()
total_time = end_time - start_time
print(total_time)
timings_file = open(filename1, "a")
timings_file.write(str(total_time) + " / " + str(epoch) + "\n")
# model_name = "./best_fcn_models/" + filename + ".pth"
# torch.save(fcn_model.state_dict(), model_name)
sys.exit()
def main():
maxIOU = 0.0
assert torch.cuda.is_available()
torch.backends.cudnn.benchmark = True
model_fname = '../data/model_focalloss_loav1/deeplabv3_{0}_epoch%d.pth'.format(
'crops')
train_dataset = CropSegmentation(train=True, crop_size=args.crop_size)
model = torchvision.models.segmentation.deeplabv3_resnet50(
pretrained=False, progress=True, num_classes=5, aux_loss=True)
if args.train:
weight = np.ones(4)
weight[2] = 5
weight[3] = 5
w = torch.FloatTensor(weight).cuda()
criterion = nn.CrossEntropyLoss() #ignore_index=255 weight=w
focal_loss = FocalLoss2d()
model = nn.DataParallel(model).cuda()
for param in model.parameters():
param.requires_grad = True
# epoch = 60
optimizer = optim.SGD(model.parameters(),
lr=args.base_lr,
momentum=0.9,
weight_decay=0.0001)
dataset_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=args.train,
pin_memory=True,
num_workers=args.workers)
max_iter = args.epochs * len(dataset_loader)
losses = AverageMeter()
start_epoch = 0
if args.resume:
if os.path.isfile(args.resume):
print('=> loading checkpoint {0}'.format(args.resume))
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print('=> loaded checkpoint {0} (epoch {1})'.format(
args.resume, checkpoint['epoch']))
else:
print('=> no checkpoint found at {0}'.format(args.resume))
for epoch in range(start_epoch, args.epochs):
# scheduler.step(epoch)
model.train()
for i, (inputs, target) in enumerate(dataset_loader):
inputs = Variable(inputs.cuda())
target = Variable(target.cuda())
outputs = model(inputs)
loss1 = focal_loss(outputs['out'], target)
loss2 = focal_loss(outputs['aux'], target)
loss01 = loss1 + 0.1 * loss2
loss3 = lovasz_softmax(outputs['out'], target)
loss4 = lovasz_softmax(outputs['aux'], target)
loss02 = loss3 + 0.1 * loss4
loss = loss01 + loss02
if np.isnan(loss.item()) or np.isinf(loss.item()):
pdb.set_trace()
losses.update(loss.item(), args.batch_size)
loss.backward()
optimizer.step()
optimizer.zero_grad()
print('epoch: {0}\t'
'iter: {1}/{2}\t'
'loss: {loss.val:.4f} ({loss.ema:.4f})'.format(
epoch + 1, i + 1, len(dataset_loader), loss=losses))
print('epoch: {0},learn rate:{1}'.format(
epoch + 1, optimizer.param_groups[0]['lr']))
if epoch % 1 == 0:
torch.save(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, model_fname % (epoch + 1))
def forward(self, input, target):
out = F.softmax(input, dim=1)
loss = L.lovasz_softmax(out, target)
return loss
def train(args, optimizer, criterion, criterion2, scheduler, train_loader, val_loader, test_loader, filename=None):
epochs = args.epochs
for epoch in range(epochs):
if args.scheduler == True:
scheduler.step()
ts = time.time()
for iter, (inputs, labels) in tqdm.tqdm(
enumerate(train_loader), total=len(train_loader),
desc='Train epoch=%d' % epoch, ncols=80, leave=False):
optimizer.zero_grad()
if use_gpu:
inputs = Variable(inputs.cuda(gpu_used))
labels = Variable(labels.cuda(gpu_used))
else:
inputs, labels = Variable(inputs), Variable(labels)
N, c, h, w = inputs.shape
intermediate_input_from_fcn = fcn_model.pretrained_net(inputs)['x4']
intermediate_input_from_fcn = transform_conv(intermediate_input_from_fcn)
outputs = fcn_model(inputs)
N, c, h, w = outputs.shape
#EXTRACT INTERMEDIATE INPUT
outputs2 = model2.decode(intermediate_input_from_fcn, iter)
if args.loss == "CE":
loss1 = criterion(outputs, labels)
loss2 = criterion2(outputs2, labels.float())
# print("main loss: ", loss1)
# print("aux loss: ", loss2)
#print(loss2)
loss = loss1 + (args.loss2weight * loss2)
else:
loss1 = L.lovasz_softmax(outputs, labels, classes=[1])
loss2 = criterion2(outputs2, labels.float())
loss = loss1 + (args.loss2weight * loss2)
if (iter+1) % 2000 == 0:
now = datetime.datetime.now()
#if args.file == True:
output1_name = "predictions-combined/" + str(now.day) + "_" + str(now.hour) + "_" + str(now.minute) + "-" + str(epoch) + "-" + str(iter) + "-FCN.jpg"
output2_name = "predictions-combined/" + str(now.day) + "_" + str(now.hour) + "_" + str(now.minute) + "-" + str(epoch) + "-" + str(iter) + "-DECODER.jpg"
targ_name = "predictions-combined/" + str(now.day) + "_" + str(now.hour) + "_" + str(now.minute) + "-" + str(epoch) + "-" + str(iter) + "-truth.jpg"
inp_name = "predictions-combined/" + str(now.day) + "_" + str(now.hour) + "_" + str(now.minute) + "-" + str(epoch) + "-" + str(iter) + "-raw.jpg"
outputs = outputs.data.cpu().numpy().transpose(0, 2, 3, 1).reshape(-1, n_class).argmax(axis=1).reshape(N, h, w)
outputs2 = outputs2.data.cpu().numpy()[:, 0, :, :]
#print(np.unique(outputs2, return_counts=True))
scipy.misc.imsave(output1_name, outputs.squeeze())
matplotlib.image.imsave(output2_name, outputs2.squeeze(), cmap='Greys_r')
#scipy.misc.imsave(output2_name, outputs2.squeeze())
scipy.misc.imsave(targ_name, labels.cpu().detach().numpy().squeeze())
scipy.misc.imsave(inp_name, inputs.cpu().detach().numpy().squeeze().transpose(1, 2, 0))
#a = list(model2.parameters())[0].clone()
loss.backward()
optimizer.step()
#b = list(model2.parameters())[0].clone()
#CONFIRMATION THAT DECODER IS FROZEN
#print("Equal?: ", torch.equal(a.data, b.data))
print("Finish epoch {}, time elapsed {}".format(epoch, time.time() - ts))
print("epoch: {}, loss: {}".format(epoch, loss.data.item()))
if args.file == True:
csv_file = open(filename, "a")
csv_file.write(str(epoch) + "," + str(loss.data.item()) + ",")
csv_file.close()
val(epoch, args, criterion, val_loader, test_loader, filename)
def symmetric_lovasz_fn(logits, target):
return lovasz_softmax(torch.softmax(logits, dim=1),
target[:, 0].long())
def train(args, optimizer, criterion, criterion2, scheduler, train_loader, val_loader, filename=None):
epochs = args.epochs
for epoch in range(epochs):
if args.scheduler == True:
scheduler.step()
ts = time.time()
for iter, (inputs, labels) in tqdm.tqdm(
enumerate(train_loader), total=len(train_loader),
desc='Train epoch=%d' % epoch, ncols=80, leave=False):
optimizer.zero_grad()
if use_gpu:
inputs = Variable(inputs.cuda(gpu_used))
labels = Variable(labels.cuda(gpu_used))
else:
inputs, labels = Variable(inputs), Variable(labels)
N, c, h, w = inputs.shape
intermediate_input_from_fcn = fcn_model.pretrained_net(inputs)['x5']
intermediate_input_from_fcn = transform_conv(intermediate_input_from_fcn)
outputs = fcn_model(inputs)
#EXTRACT INTERMEDIATE INPUT
outputs2 = model2.decode(intermediate_input_from_fcn, iter)
if args.loss == "CE":
loss1 = criterion(outputs, labels)
loss2 = criterion2(outputs2, labels.float())
# print("main loss: ", loss1)
# print("aux loss: ", loss2)
loss = loss1 + (args.loss2weight * loss2)
else:
loss1 = L.lovasz_softmax(outputs, labels, classes=[1])
loss2 = criterion2(outputs2, labels.float())
loss = loss1 + (args.loss2weight * loss2)
#a = list(model2.parameters())[0].clone()
loss.backward()
optimizer.step()
#b = list(model2.parameters())[0].clone()
#CONFIRMATION THAT DECODER IS FROZEN
#print("Equal?: ", torch.equal(a.data, b.data))
print("Finish epoch {}, time elapsed {}".format(epoch, time.time() - ts))
print("epoch: {}, loss: {}".format(epoch, loss.data.item()))
if args.file == True:
csv_file = open(filename, "a")
csv_file.write(str(epoch) + "," + str(loss.data.item()) + ",")
csv_file.close()
val(epoch, args, criterion, val_loader, filename)
def forward(self, inputs, targets):
return lovasz_softmax(self.softmax(inputs),
targets,
per_image=self.per_image,
ignore=self.ignore_index)
