def validateMaskgen(masknet, init=True):
masknet = masknet.cuda()
if init:
masknet.load_state_dict(
torch.load(DATA + 'models/' + masknet.__class__.__name__ +
'.torch'))
heatmap = rpnheatmap().cuda()
heatmap.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
criterion = nn.CrossEntropyLoss().cuda()
dataset = rpnDataset(DATA + 'dataset/rpn-valid/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
runningloss = 0
for i, data in enumerate(dataloader, 0):
targetmasks = Variable(torch.cuda.LongTensor(np.array(data['masks'])))
targetmasks.requires_grad = False
crops = Variable(torch.cuda.FloatTensor(np.array(data['crops'])))
targetmasks, crops = targetmasks / 255, crops / 255
targetmasks = targetmasks[0].view(-1, 1, 32, 32)
crops = crops[0].view(-1, 1, 32, 32)
features = heatmap(crops)
masks = masknet(features)
loss = maskgenloss(targetmasks, masks, criterion)
runningloss += loss
runningloss = runningloss / len(dataset)
print(" Mean NLL loss over validation set : ", runningloss)
pass
def validateRPN(init=True, perturb=False):
net = rpn().cuda()
if init:
net.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
dataset = rpnDataset(DATA + 'dataset/rpn-valid/', perturb=perturb)
dataset_size = len(dataset)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
criterioncls = torch.nn.CrossEntropyLoss().cuda()
criterionbbox = torch.nn.SmoothL1Loss().cuda()
loop_counter = 0
running_clsloss = 0
running_bboxloss = 0
for data in dataloader:
loop_counter += 1
image = torch.cuda.FloatTensor(np.array(data['image']))
gtbbox = data['bbox'].type(torch.cuda.FloatTensor)
classification, bboxdelta = net(Variable(image))
clstargets, bboxtargets, anchors = generateTargets(
gtbbox[0],
image.size()[-2:],
classification.size()[-2:])
clsloss = classificationloss(classification, clstargets, criterioncls)
bboxloss = bboxdeltaloss(bboxtargets, bboxdelta, clstargets, anchors,
criterionbbox)
running_clsloss += clsloss
running_bboxloss += bboxloss
print('progress {:3.3f} %, clsloss {:1.5f}'.format(
100 * loop_counter / dataset_size, clsloss.data[0]))
print('average validation clsloss {:1.5f}, bboxloss {:1.5f}'.format(
running_clsloss.data[0] / dataset_size,
running_bboxloss.data[0] / dataset_size))
pass
def validateMaskgen2(masknet, init=True, perturb=False):
masknet.cuda()
if init:
masknet.load_state_dict(
torch.load(DATA + 'models/' + masknet.__class__.__name__ +
'.torch'))
criterion = nn.CrossEntropyLoss().cuda()
dataset = rpnDataset(DATA + 'dataset/rpn-valid/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
runningloss = 0
for i, data in enumerate(dataloader, 0):
targetmasks = Variable(torch.cuda.LongTensor(np.array(data['masks'])))
targetmasks.requires_grad = False
crops = Variable(torch.cuda.FloatTensor(np.array(data['crops'])))
targetmasks, crops = targetmasks / 255, crops / 255
targetmasks = targetmasks[0].view(-1, 1, 32, 32)
crops = crops[0].view(-1, 1, 32, 32)
if perturb:
rnd = torch.rand(crops.size()).type(torch.cuda.FloatTensor)
rnd = rnd * 0.1
crops = crops + Variable(rnd)
masks = masknet(crops)
loss = maskgenloss(targetmasks, masks, criterion)
runningloss += loss
if i == 40:
break
runningloss = runningloss / 40
print("Cross entropy : {:.4f}".format(runningloss.data[0]))
pass
def trainMaskgen(epochs=4, a=0.001, b=0.1, d=8, save_model=True):
masknet = maskgen().cuda()
heatmap = rpnheatmap().cuda()
heatmap.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
for p in heatmap.parameters():
p.requires_grad = False
criterion = nn.CrossEntropyLoss().cuda()
dataset = rpnDataset(DATA + 'dataset/rpn/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
for epoch in range(epochs): # loop over the dataset multiple times
optimizer = optimize.Adam(masknet.parameters(),
lr=a * (b**(epoch // d)))
for i, data in enumerate(dataloader, 0):
targetmasks = Variable(
torch.cuda.LongTensor(np.array(data['masks'])))
targetmasks.requires_grad = False
crops = Variable(torch.cuda.FloatTensor(np.array(data['crops'])))
targetmasks, crops = targetmasks / 255, crops / 255
targetmasks = targetmasks[0].view(-1, 1, 32, 32)
crops = crops[0].view(-1, 1, 32, 32)
features = heatmap(crops)
masks = masknet(features)
loss = maskgenloss(targetmasks, masks, criterion)
loss.backward()
optimizer.step()
if i % 1 == 0:
print(
'Epoch {}, progress {:.3f}, training loss: {:.5f}'.format(
epoch + 1, 100 * i / len(dataset), loss.data[0]))
pass
print('Training finished successfully !')
if save_model:
print('Saving model to disk ... ')
if not os.path.exists(DATA + 'models/'):
os.makedirs(DATA + 'models/')
torch.save(masknet.state_dict(),
DATA + 'models/' + masknet.__class__.__name__ + '.torch')
print('Model saved to disk !')
pass
def visualizeMaskgen2():
masknet = maskgen2().cuda()
masknet.load_state_dict(
torch.load(DATA + 'models/' + masknet.__class__.__name__ + '.torch'))
threshold = nn.Sigmoid().cuda()
dataset = rpnDataset(DATA + 'dataset/rpn-valid/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
for i, data in enumerate(dataloader, 0):
targetmasks = Variable(torch.cuda.FloatTensor(np.array(data['masks'])))
crops = Variable(torch.cuda.FloatTensor(np.array(data['crops'])))
targetmasks, crops = targetmasks / 255, crops / 255
crops = crops[0].view(-1, 1, 32, 32)
targetmasks = targetmasks[0].view(-1, 1, 32, 32)
masks = masknet(crops)
masks = threshold(masks)
length = len(masks)
index = np.random.choice(length)
target = targetmasks.data.cpu()[index, 0].numpy()
mask = masks.data.cpu()[index, 0, :, :].numpy()
crop = crops.data.cpu()[index, 0, :, :].numpy()
th = 0.6
mask[mask > th] = 1
mask[mask < th] = 0
fig = plot.figure()
fig.add_subplot(1, 3, 1)
plot.axis('off')
plot.imshow(crop)
fig.add_subplot(1, 3, 2)
plot.axis('off')
plot.imshow(target)
fig.add_subplot(1, 3, 3)
plot.axis('off')
plot.imshow(mask)
plot.show()
pass
def visualizeMaskgen():
masknet = maskgen().cuda()
masknet.load_state_dict(
torch.load(DATA + 'models/' + masknet.__class__.__name__ + '.torch'))
heatmap = rpnheatmap().cuda()
heatmap.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
dataset = rpnDataset(DATA + 'dataset/rpn-valid/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
for i, data in enumerate(dataloader, 0):
targetmasks = Variable(torch.cuda.FloatTensor(np.array(data['masks'])))
crops = Variable(torch.cuda.FloatTensor(np.array(data['crops'])))
targetmasks, crops = targetmasks / 255, crops / 255
crops = crops[0].view(-1, 1, 32, 32)
targetmasks = targetmasks[0].view(-1, 1, 32, 32)
features = heatmap(crops)
masks = masknet(features)
length = len(masks)
index = np.random.choice(length)
target = targetmasks.data.cpu()[index, 0].numpy()
mask = masks.data.cpu()[index, 0, :, :].numpy()
crop = crops.data.cpu()[index, 0, :, :].numpy()
fig = plot.figure()
fig.add_subplot(1, 3, 1)
disp1 = plot.imshow(crop)
fig.add_subplot(1, 3, 2)
disp2 = plot.imshow(target)
fig.add_subplot(1, 3, 3)
disp3 = plot.imshow(mask)
plot.show()
pass
def solve():
rpnnet = rpn().cuda()
rpnnet.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
clsnet = backbone01().cuda()
clsnet.load_state_dict(torch.load(DATA + 'models/backbone01.torch'))
masknet = maskgen2().cuda()
masknet.load_state_dict(torch.load(DATA + 'models/maskgen2.torch'))
threshold = nn.Sigmoid()
dataset_path = DATA + 'dataset/rpn-valid/'
dataset = rpnDataset(dataset_path)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
for counter, data in enumerate(dataloader):
image = torch.cuda.FloatTensor(np.array(data['image']))
classification, bboxdelta = rpnnet(Variable(image))
posidx = list(range(0, 2 * len(AnchorShapes) - 1, 2))
objscore = classification[0][posidx].contiguous().view(-1)
k = len(AnchorShapes)
cxidx = list(range(0, 4 * k, 4))
cyidx = list(range(1, 4 * k, 4))
widx = list(range(2, 4 * k, 4))
hidx = list(range(3, 4 * k, 4))
bboxdelta = torch.cat((bboxdelta[0][cxidx].contiguous().view(
-1, 1), bboxdelta[0][cyidx].contiguous().view(
-1, 1), bboxdelta[0][widx].contiguous().view(
-1, 1), bboxdelta[0][hidx].contiguous().view(-1, 1)),
dim=1)
anchors = generateAnchors(image.size()[-2:],
classification.size()[-2:])
bboxdeltacpu = bboxdelta.data.cpu()
anchors = center_size(anchors)
anchors = anchors.cpu()
bboxarray = bboxdeltacpu + anchors
bboxarray = point_form(bboxarray)
objscore = objscore.data.cpu()
keep, count = nms(bboxarray, objscore, overlap=0.15, top_k=200)
bboxarray = bboxarray[keep[:count]]
bboxarray = bboxarray.numpy()
imgray = np.array(data['image'], np.uint8)[0, 0]
crop_array = np.zeros((0, 32, 32), np.uint8)
for bbox in bboxarray:
bbox = np.rint(bbox).astype(np.int)
x1, y1, x2, y2 = bbox[0], bbox[1], bbox[2], bbox[3]
x1 = 0 if x1 < 0 else x1
y1 = 0 if y1 < 0 else y1
x2 = imgray.shape[1] if x2 > imgray.shape[1] else x2
y2 = imgray.shape[0] if y2 > imgray.shape[0] else y2
if abs(x2 - x1) > 0 and abs(y2 - y1) > 0:
imcrop = imgray[y1:y2, x1:x2]
imcrop = cv2.resize(imcrop, (32, 32))
else:
imcrop = np.zeros((32, 32), np.uint8)
crop_array = np.append(crop_array,
np.expand_dims(imcrop, axis=0),
axis=0)
pass
crops = Variable(torch.cuda.FloatTensor(crop_array))
crops = crops.view(-1, 1, 32, 32)
clspred = clsnet(crops)
masks = masknet(crops)
masks = threshold(masks)
clspred = clspred.max(dim=1)[1]
masks = masks.data * clspred.data.view(clspred.size()[0], 1, 1,
1).type(torch.cuda.FloatTensor)
masks = masks.view(-1, 32, 32).cpu().numpy()
th = 0.6
masks[masks >= th] = 1
masks[masks < th] = 0
masks = masks * 255
masks = masks.astype(np.uint8)
maskunique = np.zeros(imgray.shape, np.uint8)
for i, mask in enumerate(masks, 0):
if mask.max() > 0:
maskout = np.zeros(imgray.shape, np.uint8)
bbox = np.rint(bboxarray[i]).astype(np.int)
x1, y1, x2, y2 = bbox[0], bbox[1], bbox[2], bbox[3]
x1 = 0 if x1 < 0 else x1
y1 = 0 if y1 < 0 else y1
x2 = maskout.shape[1] if x2 > maskout.shape[1] else x2
y2 = maskout.shape[0] if y2 > maskout.shape[0] else y2
if abs(x2 - x1) > 0 and abs(y2 - y1) > 0:
maskout[y1:y2,
x1:x2] = cv2.resize(mask,
(abs(x2 - x1), abs(y2 - y1)))
maskout = maskout - maskout * (maskunique / 200)
maskunique = maskunique + maskout
result_path = 'valid/'
if not os.path.exists(DATA + 'results/' + result_path +
data['id'][0] + '/'):
os.makedirs(DATA + 'results/' + result_path +
data['id'][0] + '/')
cv2.imwrite(
DATA + 'results/' + result_path + data['id'][0] + '/' +
str(i) + '.png', maskout)
if i % 10 == 0:
print("Solving ... {:.2f} %".format(100 * counter /
len(dataset)))
pass
pass
print("DONE !")
def visualizeRPN(donms=True, o=0.3, topk=200, perturb=False):
net = rpn().cuda()
net.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
# net.load_state_dict(torch.load(DATA + 'models/rpn_tp5tn1.torch'))
# net.load_state_dict(torch.load(DATA + 'models/rpn_tp7tn3.torch'))
dataset = rpnDataset(DATA + 'dataset/rpn-valid/', perturb=perturb)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
for data in dataloader:
image = torch.cuda.FloatTensor(np.array(data['image']))
gtbbox = data['bbox'].type(torch.cuda.FloatTensor)
classification, bboxdelta = net(Variable(image))
clstargets, bboxtargets, anchors = generateTargets(
gtbbox[0],
image.size()[-2:],
classification.size()[-2:])
posidx = list(range(0, 2 * len(AnchorShapes) - 1, 2))
clspos = clstargets[0][posidx].contiguous().view(-1)
objscore = classification[0][posidx].contiguous().view(-1)
k = len(AnchorShapes)
cxidx = list(range(0, 4 * k, 4))
cyidx = list(range(1, 4 * k, 4))
widx = list(range(2, 4 * k, 4))
hidx = list(range(3, 4 * k, 4))
bboxdelta = torch.cat((bboxdelta[0][cxidx].contiguous().view(
-1, 1), bboxdelta[0][cyidx].contiguous().view(
-1, 1), bboxdelta[0][widx].contiguous().view(
-1, 1), bboxdelta[0][hidx].contiguous().view(-1, 1)),
dim=1)
postargetidx = (clspos > 0).nonzero()[:, 0].cpu()
bboxdeltacpu = bboxdelta.data.cpu()
anchors = center_size(anchors)
anchors = anchors.cpu()
bboxarray = bboxdeltacpu + anchors
bboxarray = point_form(bboxarray)
objscore = objscore.data.cpu()
if donms:
keep, count = nms(bboxarray, objscore, overlap=o, top_k=topk)
bboxarray = bboxarray[keep[:count]]
print("Number of proposals : ", count)
bboxarray = bboxarray.numpy()
img = np.array(data['image'])
img = img[0, 0, :, :]
imgposanchors = np.array(img)
imgbboxpred = np.array(img)
gtbbox = gtbbox[0].cpu().numpy()
fig = plot.figure()
drawRectP(gtbbox, img)
disp1 = fig.add_subplot(1, 3, 1)
disp1.axis('off')
plot.imshow(img)
drawRectC(anchors[postargetidx], imgposanchors)
disp2 = fig.add_subplot(1, 3, 2)
disp2.axis('off')
plot.imshow(imgposanchors)
drawRectP(bboxarray, imgbboxpred)
disp3 = fig.add_subplot(1, 3, 3)
disp3.axis('off')
plot.imshow(imgbboxpred)
plot.show()
# break
pass
def trainRPN(epochs=4,
a=0.001,
b=0.1,
train_backbone=False,
init_backbone=False,
init_rpn=False,
save_model=False):
start_time = time.time()
net = rpn(train_backbone, init_backbone).cuda()
if init_rpn:
net.load_state_dict(torch.load(DATA + 'models/rpn.torch'))
dataset = rpnDataset(DATA + 'dataset/rpn/')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True)
criterioncls = torch.nn.CrossEntropyLoss().cuda()
criterionbbox = torch.nn.L1Loss().cuda()
N = 100
dataset_size = len(dataset)
clambda = 1
blambda = 10
for epoch in range(epochs):
optimizer = optimize.Adam(filter(lambda p: p.requires_grad,
net.parameters()),
lr=a * (b**epoch))
loop_counter = 0
for data in dataloader:
loop_counter += 1
image = torch.cuda.FloatTensor(np.array(data['image']))
gtbbox = data['bbox'].type(torch.cuda.FloatTensor)
classification, _ = net(Variable(image).cuda())
clstargets, bboxtargets, anchors = generateTargets(
gtbbox[0],
image.size()[-2:],
classification.size()[-2:])
for i in range(10):
optimizer.zero_grad()
classification, bboxdelta = net(Variable(image))
clsloss = classificationloss(classification, clstargets,
criterioncls, N, True)
bboxloss = bboxdeltaloss(bboxtargets, bboxdelta, clstargets,
anchors, criterionbbox)
if str(bboxloss.data[0]) != 'nan':
totalloss = clambda * clsloss + blambda * bboxloss
totalloss.backward()
optimizer.step()
print(
'Epoch {} progress {:3.3f} %, clsloss {:1.5f}, bboxloss {:1.5f}'
.format(epoch + 1, 100 * loop_counter / dataset_size,
clsloss.data[0], blambda * bboxloss.data[0]))
end_time = time.time()
print('Finished training in {:4.2f} s i.e, {:3.2f} min'.format(
end_time - start_time, (end_time - start_time) / 60))
if save_model:
print('Saving model to disk ... ')
if not os.path.exists(DATA + 'models/'):
os.makedirs(DATA + 'models/')
torch.save(net.state_dict(),
DATA + 'models/' + net.__class__.__name__ + '.torch')
print('Model saved to disk !')
pass
