def train_renderV2(model, train_dataloader, test_dataloader, n_epochs,
loss_function, date4File, cubeSetName, batch_size,
fileExtension, device, obj_name, noise, number_train_im):
# monitor loss functions as the training progresses
loop = n_epochs
Step_Val_losses = []
current_step_loss = []
current_step_Test_loss = []
Test_losses = []
Epoch_Val_losses = []
Epoch_Test_losses = []
count = 0
testcount = 0
Im2ShowGT = []
Im2ShowGCP = []
LastEpochTestCPparam = []
LastEpochTestGTparam = []
numbOfImageDataset = number_train_im
renderCount = 0
regressionCount = 0
renderbar = []
regressionbar = []
lr = 0.0001 #best at 0.0001 with translation only with span model.t[2] > 4 and model.t[2] < 8 and torch.abs(model.t[0]) < 2 and torch.abs(model.t[1]) < 2):
output_dir_model = 'models/render/{}'.format(fileExtension)
mkdir_p(output_dir_model)
output_dir_results = 'results/render/{}'.format(fileExtension)
mkdir_p(output_dir_results)
for epoch in range(n_epochs):
## Training phase
model.train()
print('train phase epoch {}/{}'.format(epoch, n_epochs))
t = tqdm(iter(train_dataloader),
leave=True,
total=len(train_dataloader))
for image, silhouette, parameter in t:
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
numbOfImage = image.size()[0]
# loss = nn.MSELoss()(params, parameter).to(device)
for i in range(0, numbOfImage):
#create and store silhouette
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
if (model.t[2] > 4 and model.t[2] < 8
and torch.abs(model.t[0]) < 2
and torch.abs(model.t[1]) < 2):
# if (epoch > 0):
# optimizer = torch.optim.SGD(model.parameters(), lr=lr)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
optimizer.zero_grad()
if (i == 0):
loss = nn.BCELoss()(current_sil,
current_GT_sil).to(device)
else:
loss = loss + nn.BCELoss()(current_sil,
current_GT_sil).to(device)
renderCount += 1
else:
# optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
optimizer.zero_grad()
if (i == 0):
loss = nn.MSELoss()(params[i, 3:6],
parameter[i, 3:6]).to(device)
# loss = nn.MSELoss()(params[i], parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(
params[i, 3:6], parameter[i, 3:6]).to(device)
# loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
regressionCount += 1
loss.backward()
optimizer.step()
Step_Val_losses.append(loss.detach().cpu().numpy()
) # contain all step value for all epoch
current_step_loss.append(loss.detach().cpu().numpy(
)) # contain only this epoch loss, will be reset after each epoch
count = count + 1
# if (epoch % 5 == 0 and epoch > 2):
# if (lr > 0.000001):
# lr = lr / 10
# print('update lr, is now {}'.format(lr))
epochValloss = np.mean(current_step_loss)
current_step_loss = []
Epoch_Val_losses.append(
epochValloss) # most significant value to store
# print(epochValloss)
print(Epoch_Val_losses)
# print(renderCount, regressionCount)
renderbar.append(renderCount)
regressionbar.append(regressionCount)
renderCount = 0
regressionCount = 0
torch.save(
model.state_dict(),
'{}/{}_{}epoch_{}_Temp_train_{}_{}batchs_{}epochs_Render.pth'.
format(
output_dir_model,
fileExtension,
epoch,
date4File,
cubeSetName,
str(batch_size),
str(n_epochs),
))
# validation phase
print('test phase epoch epoch {}/{}'.format(epoch, n_epochs))
model.eval()
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
# print(np.shape(params))
for i in range(0, numbOfImage):
#create and store silhouette
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
if (i == 0):
loss = nn.BCELoss()(current_sil, current_GT_sil).to(device)
else:
loss = loss + nn.BCELoss()(current_sil,
current_GT_sil).to(device)
Test_Step_loss.append(loss.detach().cpu().numpy())
if (epoch == n_epochs - 1
): # if we are at the last epoch, save param to plot result
LastEpochTestCPparam.extend(params.detach().cpu().numpy())
LastEpochTestGTparam.extend(parameter.detach().cpu().numpy())
Test_losses.append(loss.detach().cpu().numpy())
current_step_Test_loss.append(loss.detach().cpu().numpy())
testcount = testcount + 1
epochTestloss = np.mean(current_step_Test_loss)
current_step_Test_loss = []
Epoch_Test_losses.append(
epochTestloss) # most significant value to store
# ----------- plot some result from the last epoch computation ------------------------
# print(np.shape(LastEpochTestCPparam)[0])
nim = 4
for i in range(0, nim):
print('saving image to show')
pickim = int(uniform(0, np.shape(LastEpochTestCPparam)[0] - 1))
# print(pickim)
model.t = torch.from_numpy(
LastEpochTestCPparam[pickim][3:6]).to(device)
R = torch.from_numpy(LastEpochTestCPparam[pickim][0:3]).to(device)
model.R = R2Rmat(R) # angle from resnet are in radia
imgCP, _, _ = model.renderer(model.vertices,
model.faces,
torch.tanh(model.textures),
R=model.R,
t=model.t)
model.t = torch.from_numpy(
LastEpochTestGTparam[pickim][3:6]).to(device)
R = torch.from_numpy(LastEpochTestGTparam[pickim][0:3]).to(device)
model.R = R2Rmat(R) # angle from resnet are in radia
imgGT, _, _ = model.renderer(model.vertices,
model.faces,
torch.tanh(model.textures),
R=model.R,
t=model.t)
imgCP = imgCP.squeeze() # float32 from 0-1
imgCP = imgCP.detach().cpu().numpy().transpose((1, 2, 0))
imgCP = (imgCP * 255).astype(
np.uint8) # cast from float32 255.0 to 255 uint8
imgGT = imgGT.squeeze() # float32 from 0-1
imgGT = imgGT.detach().cpu().numpy().transpose((1, 2, 0))
imgGT = (imgGT * 255).astype(
np.uint8) # cast from float32 255.0 to 255 uint8
Im2ShowGT.append(imgCP)
Im2ShowGCP.append(imgGT)
a = plt.subplot(2, nim, i + 1)
plt.imshow(imgGT)
a.set_title('GT {}'.format(i))
plt.xticks([0, 512])
plt.yticks([])
a = plt.subplot(2, nim, i + 1 + nim)
plt.imshow(imgCP)
a.set_title('Rdr {}'.format(i))
plt.xticks([0, 512])
plt.yticks([])
plt.savefig('{}/image_render_{}batch_{}epochs_{}.png'.format(
output_dir_results, batch_size, n_epochs, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
#-----------plot and save section ------------------------------------------------------------------------------------
fig, (p1, p2, p3, p4) = plt.subplots(4,
figsize=(15, 10)) # largeur hauteur
fig.suptitle("Render, training {} epochs with {} images, lr={} ".format(
n_epochs, numbOfImageDataset, lr),
fontsize=14)
moving_aves = RolAv(Step_Val_losses, window=50)
ind = np.arange(n_epochs) # index
p1.plot(np.arange(np.shape(moving_aves)[0]),
moving_aves,
label="step Loss rolling average")
p1.set(ylabel='BCE Step Loss')
p1.set_yscale('log')
p1.set(xlabel='Steps')
p1.set_ylim([0, 20])
p1.legend() # Place a legend to the right of this smaller subplot.
# subplot 2
p2.plot(np.arange(n_epochs), Epoch_Val_losses, label="Render epoch Loss")
p2.set(ylabel=' Mean of BCE training step loss')
p2.set(xlabel='Epochs')
p2.set_ylim([0, 20])
p2.set_xticks(ind)
p2.legend()
# subplot 3
width = 0.35
p3.bar(ind, renderbar, width, color='#d62728', label="render")
height_cumulative = renderbar
p3.bar(ind,
regressionbar,
width,
bottom=height_cumulative,
label="regression")
p3.set(ylabel='render/regression call')
p3.set(xlabel='Epochs')
p3.set_ylim([0, numbOfImageDataset])
p3.set_xticks(ind)
p3.legend()
# subplot 4
p4.plot(np.arange(n_epochs), Epoch_Test_losses, label="Render Test Loss")
p4.set(ylabel='Mean of BCE test step loss')
p4.set(xlabel='Epochs')
p4.set_ylim([0, 10])
p4.legend()
plt.show()
fig.savefig('{}/render_{}batch_{}epochs_{}.png'.format(
output_dir_results, batch_size, n_epochs, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
matplotlib2tikz.save("{}/render_{}batch_{}epochs_{}.tex".format(
output_dir_results, batch_size, n_epochs, fileExtension),
figureheight='5.5cm',
figurewidth='15cm')
torch.save(
model.state_dict(),
'{}/{}_{}_FinalModel_train_{}_{}batchs_{}epochs_Render.pth'.format(
output_dir_model,
fileExtension,
date4File,
cubeSetName,
str(batch_size),
str(n_epochs),
))
print('parameters saved')
def Val_V1(model, val_dataloader, n_epochs, fileExtension, device, traintype,
lr, validation, useofFK, ResnetOutput, SettingString,
useOwnPretrainedModel):
output_result_dir = 'results/Validation_{}{}/{}'.format(
traintype, ResnetOutput, fileExtension)
mkdir_p(output_result_dir)
current_dir = os.path.dirname(os.path.realpath(__file__))
sil_dir = os.path.join(output_result_dir)
parser = argparse.ArgumentParser()
parser.add_argument('-or',
'--filename_output',
type=str,
default=os.path.join(
sil_dir,
'ValidationGif_{}.gif'.format('LongShaft_2')))
parser.add_argument('-mr', '--make_reference_image', type=int, default=0)
parser.add_argument('-g', '--gpu', type=int, default=0)
args = parser.parse_args()
# validation --------------------------------------------------------------------------------------------------------
print('validation phase')
Valcount = 0
processcount = 0
step_Val_loss = []
Epoch_Val_losses = []
model.eval()
from PIL import ImageTk, Image, ImageDraw
paramList = open("./{}/paramList.txt".format(output_result_dir), "w+")
t = tqdm(iter(val_dataloader), leave=True, total=len(val_dataloader))
for image, silhouette, parameter in t:
i = 0
Test_Step_loss = []
numbOfImage = image.size()[0]
# image1 = torch.flip(image,[0, 3]) #flip vertical
# image = torch.roll(image, 100, 3) #shift down from 100 px
image1 = shiftPixel(image, 50, 'y')
image1 = shiftPixel(image1, 50, 'x')
# image1 = torch.flip(image1, [0, 3])
# image1 = image
Origimagesave = image1
# Origimagesave = image.to(device)
image1 = image1.to(device) #torch.Size([1, 3, 1024, 1280])
parameter = parameter.to(device)
#parameter estimation through trained model
if ResnetOutput == 't': # resnet predict only translation parameter
print('own model used is t')
t_params = model(image1)
model.t = t_params[i]
R = parameter[
i,
0:3] # give the ground truth parameter for the rotation values
model.R = R2Rmat(R)
paramList.write('step:{} params:{} \r\n'.format(
processcount,
t_params.detach().cpu().numpy()))
loss = nn.MSELoss()(t_params[i], parameter[i, 3:6]).to(device)
if ResnetOutput == 'Rt': # resnet predict rotation and translation
params = model(image1)
print('own model used is Rt')
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
paramList.write('step:{} params:{} \r\n'.format(
processcount,
params.detach().cpu().numpy()))
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
i = 0
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
sil2plot = np.squeeze(
(current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
image2show = np.squeeze((Origimagesave[i].detach().cpu().numpy()))
image2show = (image2show * 0.5 + 0.5).transpose(1, 2, 0)
# image2show = np.flip(image2show,1)
# fig = plt.figure()
# fig.add_subplot(2, 1, 1)
# plt.imshow(sil2plot, cmap='gray')
#
# fig.add_subplot(2, 1, 2)
# plt.imshow(image2show)
# plt.show()
#creation of the blender image
sil3d = sil2plot[:, :, np.newaxis]
renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2)
toolIm = Image.fromarray(np.uint8(renderim))
# print(type(image2show))
backgroundIm = Image.fromarray(np.uint8(image2show * 255))
# backgroundIm.show()
#
alpha = 0.2
out = Image.blend(backgroundIm, toolIm, alpha)
# #make gif
imsave('/tmp/_tmp_%04d.png' % processcount, np.array(out))
processcount = processcount + 1
step_Val_loss.append(loss.detach().cpu().numpy())
# print(processcount)
print('making the gif')
make_gif(args.filename_output)
print(step_Val_loss)
print(np.mean(step_Val_loss))
paramList.close()
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
# print(np.shape(params))
for i in range(0, numbOfImage):
#create and store silhouette
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
imgCP, _, _ = model.renderer(model.vertices,
model.faces,
torch.tanh(model.textures),
R=model.R,
t=model.t)
imgGT = image[i]
def training(model, train_dataloader, test_dataloader, val_dataloader,
n_epochs, fileExtension, device, traintype, lr, validation,
number_test_im, useofFK, ResnetOutput, SettingString,
useOwnPretrainedModel):
# monitor loss functions as the training progresses
current_step_Train_loss = []
current_step_Train_MSEloss = []
Epoch_Train_losses = []
Epoch_Train_MSElosses = []
count = 0
epoch_count = 1
lr = lr
print('training {}'.format(traintype))
print('lr used is: {}'.format(lr))
output_result_dir = 'results/{}{}/{}_lr{}'.format(traintype, ResnetOutput,
fileExtension, lr)
mkdir_p(output_result_dir)
epochsTrainLoss = open(
"{}/epochsTrainLoss_{}.txt".format(output_result_dir, fileExtension),
"w+")
epochsTrainMSELoss = open(
"{}/epochsTrainMSELoss_{}.txt".format(output_result_dir,
fileExtension), "w+")
TestParamLoss = open(
"{}/TestParamLoss_{}.txt".format(output_result_dir, fileExtension),
"w+")
ExperimentSettings = open(
"{}/expSettings_{}.txt".format(output_result_dir, fileExtension), "w+")
ExperimentSettings.write(SettingString)
ExperimentSettings.close()
x = np.arange(n_epochs)
div = 0.8
slope = 0.5
y = sigmoid(x, 1, 0, n_epochs / div, slope)
plt.plot(x, y)
# plt.title('div:{} slope:{}'.format(div,slope))
plt.xlabel('epochs')
plt.ylabel('\u03B1')
plt.savefig('{}/ReverseSigmoid_{}.png'.format(output_result_dir,
fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
#usefull for the loss plot of each parameter
epoch_test_loss = []
epoch_test_alpha_loss = []
epoch_test_beta_loss = []
epoch_test_gamma_loss = []
epoch_test_x_loss = []
epoch_test_y_loss = []
epoch_test_z_loss = []
Epoch_Test_losses = []
for epoch in range(n_epochs):
## training phase --------------------------------------------------------------------------------------------------------
model.train()
print('train phase epoch {}/{}'.format(epoch, n_epochs))
if useofFK:
alpha = -1 #y[epoch] #combination of the ground truth with the computed values
else:
if useOwnPretrainedModel:
alpha = 0 #if we continue to train a existing model, we want to train it for pure renderer
else:
alpha = y[epoch]
print('alpha is {}'.format(alpha))
t = tqdm(iter(train_dataloader),
leave=True,
total=len(train_dataloader))
for image, silhouette, parameter in t:
image = image.to(device)
if useofFK:
FKparameters = FKBuild(
parameter, np.radians(2), 0.001
) #add noise to the ground truth parameter, degree and cm
FKparameters = FKparameters.to(device)
params = model(image, FKparameters,
useofFK) # should be size [batchsize, 6]
else:
if ResnetOutput == 'Rt':
params = model(image) #call the 6 parameters model
if ResnetOutput == 't':
t_params = model(image) #call the 3 parameters model
# print(t_params.size())
parameter = parameter.to(device) #ground truth parameter
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
optimizer.zero_grad()
numbOfImage = image.size()[0]
mseLoss = 0 #store the mse loss
for i in range(0, numbOfImage):
if ResnetOutput == 'Rt':
print('Renset output 6 values')
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R)
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
if (traintype == 'render'):
if useofFK: #use of the mlp
print('render with FK for Rt')
if (i == 0):
loss = nn.BCELoss()(current_sil,
current_GT_sil).to(device)
else:
loss += nn.BCELoss()(current_sil,
current_GT_sil).to(device)
else: #use sigmoid curve
if (i == 0):
print('render with sigmoid for Rt')
loss = (nn.BCELoss()(current_sil,
current_GT_sil).to(device)
) * (1 - alpha) + (nn.MSELoss()(
params[i],
parameter[i]).to(device)) * (alpha)
else:
loss += (nn.BCELoss()(
current_sil, current_GT_sil).to(
device)) * (1 - alpha) + (nn.MSELoss()(
params[i],
parameter[i]).to(device)) * (alpha)
if (traintype == 'regression'):
print('regression for Rt')
if (i == 0):
loss = nn.MSELoss()(params[i],
parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(params[i],
parameter[i]).to(device)
print(loss)
if ResnetOutput == 't':
print('Resnet output 3 values')
# t_params[i][0]= parameter[i, 4] #overrride x and y
# t_params[i][1] = parameter[i, 5]
model.t = t_params[i]
print(model.t)
R = parameter[
i, 0:
3] #give the ground truth parameter for the rotation values
model.R = R2Rmat(R)
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
if (traintype == 'render'):
print('t_param is {}'.format(t_params[i]))
print('Gt_param is {}'.format(parameter[i, 3:6]))
if useofFK: # use of the mlp
print('render with FK for t')
if (i == 0):
loss = nn.BCELoss()(current_sil,
current_GT_sil).to(device)
else:
loss += nn.BCELoss()(current_sil,
current_GT_sil).to(device)
else: # use sigmoid curve
print('render with sigmoid for t')
if (i == 0):
loss = (nn.BCELoss()(
current_sil, current_GT_sil
).to(device)) * (1 - alpha) + (nn.MSELoss()(
t_params[i],
parameter[i, 3:6]).to(device)) * (alpha)
mseLoss = nn.MSELoss()(
t_params[i], parameter[i, 3:6]).to(device)
else:
loss += (nn.BCELoss()(
current_sil, current_GT_sil
).to(device)) * (1 - alpha) + (nn.MSELoss()(
t_params[i],
parameter[i, 3:6]).to(device)) * (alpha)
mseLoss += nn.MSELoss()(
t_params[i], parameter[i, 3:6]).to(device)
if (traintype == 'regression'):
print('regression for t')
if (i == 0):
print('t_param is {}'.format(t_params[i]))
print('Gt_param is {}'.format(parameter[i, 3:6]))
loss = nn.MSELoss()(t_params[i],
parameter[i, 3:6]).to(device)
else:
loss = loss + nn.MSELoss()(
t_params[i], parameter[i, 3:6]).to(device)
loss = loss / numbOfImage
mseLoss = mseLoss / numbOfImage
print('number of image is {}'.format(numbOfImage))
print('step {} loss is {}'.format(count, loss))
print('step {} MSE loss is {}'.format(count, mseLoss))
loss.backward()
optimizer.step()
current_step_Train_loss.append(loss.detach().cpu().numpy(
)) # contain only this epoch loss, will be reset after each epoch
current_step_Train_MSEloss.append(mseLoss.detach().cpu().numpy())
count = count + 1
epochTrainloss = np.mean(current_step_Train_loss)
epochTrainMSELoss = np.mean(current_step_Train_MSEloss)
epochsTrainLoss.write(
'step: {}/{} current step loss: {:.4f}\r\n'.format(
epoch, n_epochs, epochTrainloss))
epochsTrainMSELoss.write(
'step: {}/{} current step MSE loss: {:.4f}\r\n'.format(
epoch, n_epochs, epochTrainMSELoss))
print('loss of epoch {} is {}'.format(epoch, epochTrainloss))
print('MSE loss of epoch {} is {}'.format(epoch, epochTrainMSELoss))
# save the model
output_model_dir = '{}/modelTemp'.format(output_result_dir)
mkdir_p(output_model_dir)
torch.save(
model.state_dict(),
'{}/TempModel_train_{}{}.pth'.format(output_model_dir,
fileExtension, count))
current_step_Train_loss = [] #reset value
current_step_Train_MSEloss = []
Epoch_Train_losses.append(
epochTrainloss) # most significant value to store
Epoch_Train_MSElosses.append(epochTrainMSELoss)
## test phase --------------------------------------------------------------------------------------------------------
count = 0
testcount = 0
model.eval()
current_step_Test_loss = []
steps_losses = [] # reset the list after each epoch
steps_alpha_loss = []
steps_beta_loss = []
steps_gamma_loss = []
steps_x_loss = []
steps_y_loss = []
steps_z_loss = []
for i in range(number_test_im):
output_test_image_dir = '{}/images/testIm{}'.format(
output_result_dir, i)
mkdir_p(output_test_image_dir)
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
image = image.to(device)
parameter = parameter.to(device)
if ResnetOutput == 'Rt':
params = model(image)
if ResnetOutput == 't':
t_params = model(image) # should be
# print(t_params.size())
# params = model(image, 0, False) # should be size [batchsize, 6]
# print(np.shape(params))
for i in range(0, numbOfImage):
if ResnetOutput == 'Rt':
print('test for Rt')
print('image tested: {}'.format(testcount))
print('estimated {}'.format(params[i]))
print('Ground Truth {}'.format(parameter[i]))
if (i == 0):
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(params[i],
parameter[i]).to(device)
# one value each for the step, compute mse loss for all parameters separately
alpha_loss = nn.MSELoss()(
params[:, 0], parameter[:, 0]).detach().cpu().numpy()
beta_loss = nn.MSELoss()(
params[:, 1], parameter[:, 1]).detach().cpu().numpy()
gamma_loss = nn.MSELoss()(
params[:, 2], parameter[:, 2]).detach().cpu().numpy()
x_loss = nn.MSELoss()(params[:, 3],
parameter[:,
3]).detach().cpu().numpy()
y_loss = nn.MSELoss()(params[:, 4],
parameter[:,
4]).detach().cpu().numpy()
z_loss = nn.MSELoss()(params[:, 5],
parameter[:,
5]).detach().cpu().numpy()
steps_losses.append(
loss.item()) # only one loss value is add each step
steps_alpha_loss.append(alpha_loss.item())
steps_beta_loss.append(beta_loss.item())
steps_gamma_loss.append(gamma_loss.item())
steps_x_loss.append(x_loss.item())
steps_y_loss.append(y_loss.item())
steps_z_loss.append(z_loss.item())
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
if ResnetOutput == 't':
print('test for t')
print('image tested: {}'.format(testcount))
print('estimated {}'.format(t_params[i]))
print('Ground Truth {}'.format(parameter[i, 3:6]))
if (i == 0):
loss = nn.MSELoss()(t_params[i],
parameter[i, 3:6]).to(device)
else:
loss = loss + nn.MSELoss()(
t_params[i], parameter[i, 3:6]).to(device)
# one value each for the step, compute mse loss for all parameters separately
alpha_loss = 0
beta_loss = 0
gamma_loss = 0
x_loss = nn.MSELoss()(t_params[:, 0],
parameter[:,
3]).detach().cpu().numpy()
y_loss = nn.MSELoss()(t_params[:, 1],
parameter[:,
4]).detach().cpu().numpy()
z_loss = nn.MSELoss()(t_params[:, 2],
parameter[:,
5]).detach().cpu().numpy()
steps_losses.append(
loss.item()) # only one loss value is add each step
steps_alpha_loss.append(0)
steps_beta_loss.append(0)
steps_gamma_loss.append(0)
steps_x_loss.append(x_loss.item())
steps_y_loss.append(y_loss.item())
steps_z_loss.append(z_loss.item())
model.t = t_params[i]
R = parameter[
i, 0:
3] #give the ground truth parameter for the rotation values
model.R = R2Rmat(R)
print('Rt test GTparameter are {}'.format(parameter))
print('Rt test parameter are {}{}'.format(R, model.t))
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
sil2plot = np.squeeze(
(current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
fig = plt.figure()
fig.add_subplot(2, 1, 1)
plt.imshow(sil2plot, cmap='gray')
fig.add_subplot(2, 1, 2)
plt.imshow(silhouette[i], cmap='gray')
plt.savefig('{}/images/testIm{}/im{}epoch{}.png'.format(
output_result_dir, testcount, testcount, epoch),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
#MSE loss
current_step_Test_loss.append(loss.detach().cpu().numpy())
testcount = testcount + 1
#loop here for each epoch
epoch_test_loss.append(np.mean(steps_losses))
epoch_test_alpha_loss.append(np.mean(steps_alpha_loss))
epoch_test_beta_loss.append(np.mean(steps_beta_loss))
epoch_test_gamma_loss.append(np.mean(steps_gamma_loss))
epoch_test_x_loss.append(np.mean(steps_x_loss))
epoch_test_y_loss.append(np.mean(steps_y_loss))
epoch_test_z_loss.append(np.mean(steps_z_loss))
TestParamLoss.write(
'test epoch: {} current step loss: {:.7f}, angle loss: {:.4f} {:.4f} {:.4f} translation loss: {:.7f} {:.7f} {:.7f}\r\n'
.format(epoch, np.mean(steps_losses), np.mean(steps_alpha_loss),
np.mean(steps_beta_loss), np.mean(steps_gamma_loss),
np.mean(steps_x_loss), np.mean(steps_z_loss),
np.mean(steps_y_loss)))
epochTestloss = np.mean(current_step_Test_loss)
Epoch_Test_losses.append(
epochTestloss) # most significant value to store
epoch_count = epoch_count + 1
fig, (ax1, ax11, ax2) = plt.subplots(3, 1)
ax1.semilogy(Epoch_Train_losses)
if traintype == 'render':
if useOwnPretrainedModel:
ax1.set_ylabel('train {} BCE loss'.format(traintype))
else:
ax1.set_ylabel('train {} BCE/MSE loss'.format(traintype))
else:
ax1.set_ylabel('train {} MSE loss'.format(traintype))
ax1.set_xlim([0, n_epochs])
# ax1.set_ylim([0, 4])
# ax1.set_yscale('log')
ax11.semilogy(Epoch_Train_MSElosses)
ax11.set_ylabel('train {} MSE loss'.format(traintype))
ax11.set_xlim([0, n_epochs])
ax2.semilogy(Epoch_Test_losses)
ax2.set_ylabel('test {} loss'.format(traintype))
ax2.set_xlabel('epoch')
ax2.set_xlim([0, n_epochs])
# ax2.set_ylim([0, 0.1])
# ax2.set_yscale('log')
plt.savefig('{}/training_epochs_{}_results_{}.png'.format(
output_result_dir, traintype, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
fig, (a, b, g) = plt.subplots(3, 1)
a.semilogy(epoch_test_alpha_loss)
a.set_xlim([0, n_epochs])
a.set_ylabel('a')
b.semilogy(epoch_test_beta_loss)
b.set_xlim([0, n_epochs])
b.set_ylabel('b')
g.semilogy(epoch_test_gamma_loss)
g.set_xlim([0, n_epochs])
g.set_ylabel('g')
g.set_xlabel('MSE loss evolution through epochs')
plt.savefig('{}/test_epochs_{}_R_Loss_{}.png'.format(
output_result_dir, traintype, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
fig, (x, y, z) = plt.subplots(3, 1)
x.semilogy(epoch_test_x_loss)
x.set_xlim([0, n_epochs])
x.set_ylabel('x')
y.semilogy(epoch_test_y_loss)
y.set_xlim([0, n_epochs])
y.set_ylabel('y')
z.semilogy(epoch_test_z_loss)
z.set_xlim([0, n_epochs])
z.set_ylabel('z')
z.set_xlabel('MSE loss evolution through epochs')
plt.savefig('{}/test_epochs_{}_t_Loss_{}.png'.format(
output_result_dir, traintype, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
# save the model
output_model_dir = '{}/model'.format(output_result_dir)
mkdir_p(output_model_dir)
torch.save(
model.state_dict(),
'{}/FinalModel_train_{}.pth'.format(output_model_dir, fileExtension))
print('parameters saved')
epochsTrainLoss.close()
TestParamLoss.close()
def train_regressionV2(model, train_dataloader, test_dataloader, n_epochs,
loss_function, date4File, cubeSetName, batch_size,
fileExtension, device, obj_name, noise,
number_train_im):
# monitor loss functions as the training progresses
lr = 0.001
loop = n_epochs
Step_Val_losses = []
current_step_loss = []
current_step_Test_loss = []
Test_losses = []
Epoch_Val_losses = []
Epoch_Test_losses = []
count = 0
testcount = 0
Im2ShowGT = []
Im2ShowGCP = []
LastEpochTestCPparam = []
LastEpochTestGTparam = []
numbOfImageDataset = number_train_im
for epoch in range(n_epochs):
## Training phase
model.train()
print('train phase epoch {}/{}'.format(epoch, n_epochs))
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
t = tqdm(iter(train_dataloader),
leave=True,
total=len(train_dataloader))
for image, silhouette, parameter in t:
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
optimizer.zero_grad()
numbOfImage = image.size()[0]
# loss = nn.MSELoss()(params, parameter).to(device)
for i in range(0, numbOfImage):
#create and store silhouette
if (i == 0):
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(params[i],
parameter[i]).to(device)
loss.backward()
optimizer.step()
# print(loss)
Step_Val_losses.append(loss.detach().cpu().numpy()
) # contain all step value for all epoch
current_step_loss.append(loss.detach().cpu().numpy(
)) # contain only this epoch loss, will be reset after each epoch
count = count + 1
epochValloss = np.mean(current_step_loss)
current_step_loss = []
Epoch_Val_losses.append(
epochValloss) # most significant value to store
torch.save(
model.state_dict(),
'models/{}_{}epoch_{}_TempModel_train_{}_{}batchs_{}epochs_Noise{}_Regression.pth'
.format(
fileExtension,
epoch,
date4File,
cubeSetName,
str(batch_size),
str(n_epochs),
noise * 100,
))
# print(epochValloss)
# validation phase
print('test phase epoch epoch {}/{}'.format(epoch, n_epochs))
model.eval()
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
# print(np.shape(params))
for i in range(0, numbOfImage):
if (i == 0):
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(params[i],
parameter[i]).to(device)
Test_Step_loss.append(loss.detach().cpu().numpy())
if (epoch == n_epochs - 1
): # if we are at the last epoch, save param to plot result
LastEpochTestCPparam.extend(params.detach().cpu().numpy())
LastEpochTestGTparam.extend(parameter.detach().cpu().numpy())
Test_losses.append(loss.detach().cpu().numpy())
current_step_Test_loss.append(loss.detach().cpu().numpy())
testcount = testcount + 1
epochTestloss = np.mean(current_step_Test_loss)
current_step_Test_loss = []
Epoch_Test_losses.append(
epochTestloss) # most significant value to store
# ----------- plot some result from the last epoch computation ------------------------
# print(np.shape(LastEpochTestCPparam)[0])
nim = 5
for i in range(0, nim):
print('saving image to show')
pickim = int(uniform(0, np.shape(LastEpochTestCPparam)[0] - 1))
# print(pickim)
model.t = torch.from_numpy(
LastEpochTestCPparam[pickim][3:6]).to(device)
R = torch.from_numpy(LastEpochTestCPparam[pickim][0:3]).to(device)
model.R = R2Rmat(R) # angle from resnet are in radia
imgCP, _, _ = model.renderer(model.vertices,
model.faces,
torch.tanh(model.textures),
R=model.R,
t=model.t)
model.t = torch.from_numpy(
LastEpochTestGTparam[pickim][3:6]).to(device)
R = torch.from_numpy(LastEpochTestGTparam[pickim][0:3]).to(device)
model.R = R2Rmat(R) # angle from resnet are in radia
imgGT, _, _ = model.renderer(model.vertices,
model.faces,
torch.tanh(model.textures),
R=model.R,
t=model.t)
imgCP = imgCP.squeeze() # float32 from 0-1
imgCP = imgCP.detach().cpu().numpy().transpose((1, 2, 0))
imgCP = (imgCP * 255).astype(
np.uint8) # cast from float32 255.0 to 255 uint8
imgGT = imgGT.squeeze() # float32 from 0-1
imgGT = imgGT.detach().cpu().numpy().transpose((1, 2, 0))
imgGT = (imgGT * 255).astype(
np.uint8) # cast from float32 255.0 to 255 uint8
Im2ShowGT.append(imgCP)
Im2ShowGCP.append(imgGT)
a = plt.subplot(2, nim, i + 1)
plt.imshow(imgGT)
a.set_title('GT {}'.format(i))
plt.xticks([0, 512])
plt.yticks([])
a = plt.subplot(2, nim, i + 1 + nim)
plt.imshow(imgCP)
a.set_title('Rdr {}'.format(i))
plt.xticks([0, 512])
plt.yticks([])
plt.savefig('results/image_regression_{}batch_{}_{}.pdf'.format(
batch_size, n_epochs, fileExtension))
# -----------plot and save section ------------------------------------------------------------------------------------
fig, (p1, p2, p4) = plt.subplots(3, figsize=(15, 10)) # largeur hauteur
moving_aves = RolAv(Step_Val_losses, window=20)
p1.plot(np.arange(np.shape(moving_aves)[0]),
moving_aves,
label="step Loss rolling average")
p1.set(ylabel='BCE Step Loss')
p1.set_yscale('log')
p1.set(xlabel='Steps')
p1.set_ylim([0, 4])
p1.legend() # Place a legend to the right of this smaller subplot.
# subplot 2
p2.plot(np.arange(n_epochs), Epoch_Val_losses, label="epoch Loss")
p2.set(ylabel=' Mean of BCE training step loss')
p2.set(xlabel='Epochs')
p2.set_ylim([0, 4])
p2.legend()
p4.plot(np.arange(n_epochs), Epoch_Test_losses, label="Test Loss")
p4.set(ylabel='Mean of BCE test step loss')
p4.set(xlabel='Epochs')
p4.set_ylim([0, 2])
p4.legend()
plt.show()
fig.savefig('results/regression_{}batch_{}_{}.pdf'.format(
batch_size, n_epochs, fileExtension))
import matplotlib2tikz
matplotlib2tikz.save("results/regression_{}batch_{}_{}.tex".format(
batch_size, n_epochs, fileExtension))
def train_regV3(model, train_dataloader, test_dataloader, n_epochs,
loss_function, date4File, cubeSetName, batch_size,
fileExtension, device, obj_name, noise, number_train_im,
val_dataloader):
# monitor loss functions as the training progresses
loop = n_epochs
Step_Val_losses = []
current_step_loss = []
current_step_Test_loss = []
Test_losses = []
Epoch_Val_losses = []
allstepvalloss = []
Epoch_Test_losses = []
count = 0
epoch_count = 1
testcount = 0
Im2ShowGT = []
Im2ShowGCP = []
LastEpochTestCPparam = []
LastEpochTestGTparam = []
numbOfImageDataset = number_train_im
renderCount = 0
regressionCount = 0
renderbar = []
regressionbar = []
lr = 0.00001
# training -------------------------------------------------------------------------------------------------------
# for epoch in range(n_epochs):
#
# ## Training phase
# model.train()
# print('train phase epoch {}/{}'.format(epoch, n_epochs))
# optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
#
# t = tqdm(iter(train_dataloader), leave=True, total=len(train_dataloader))
# for image, silhouette, parameter in t:
# image = image.to(device)
# parameter = parameter.to(device)
# params = model(image) # should be size [batchsize, 6]
# optimizer.zero_grad()
#
# # print(params)
# numbOfImage = image.size()[0]
# # loss = nn.MSELoss()(params, parameter).to(device)
#
#
# for i in range(0,numbOfImage):
# #create and store silhouette
# if (i == 0):
# loss =nn.MSELoss()(params[i], parameter[i]).to(device)
# else:
# loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
#
#
# loss.backward()
# optimizer.step()
#
# print(loss)
# current_step_loss.append(loss.detach().cpu().numpy()) # contain only this epoch loss, will be reset after each epoch
# count = count + 1
#
# epochValloss = np.mean(current_step_loss)
# current_step_loss = [] #reset value
# Epoch_Val_losses.append(epochValloss) # most significant value to store
# # print(epochValloss)
#
# print(Epoch_Val_losses)
#
#
# #test --------------------------------------------------------------------------------------------------------
#
# count = 0
# testcount = 0
# model.eval()
#
# t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
# for image, silhouette, parameter in t:
#
# Test_Step_loss = []
# numbOfImage = image.size()[0]
#
# image = image.to(device)
# parameter = parameter.to(device)
# params = model(image) # should be size [batchsize, 6]
# # print(np.shape(params))
#
# for i in range(0, numbOfImage):
# print('image tested: {}'.format(testcount))
# print('estated {}'.format(params[i]))
# print('Ground Truth {}'.format(parameter[i]))
# if (i == 0):
# loss = nn.MSELoss()(params[i], parameter[i]).to(device)
# else:
# loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
#
# if epoch_count == n_epochs:
# model.t = params[i, 3:6]
# R = params[i, 0:3]
# model.R = R2Rmat(R) # angle from resnet are in radian
#
# current_sil = model.renderer(model.vertices, model.faces, R=model.R, t=model.t,
# mode='silhouettes').squeeze()
# current_sil = current_sil[0:1024, 0:1280]
# sil2plot = np.squeeze((current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
# current_GT_sil = (silhouette[i] / 255).type(torch.FloatTensor).to(device)
#
# fig = plt.figure()
# fig.add_subplot(2, 1, 1)
# plt.imshow(sil2plot, cmap='gray')
#
# fig.add_subplot(2, 1, 2)
# plt.imshow(silhouette[i], cmap='gray')
# plt.savefig('results/image_{}.png'.format(testcount), bbox_inches='tight',
# pad_inches=0.05)
# # plt.show()
#
# current_step_Test_loss.append(loss.detach().cpu().numpy())
# testcount = testcount + 1
#
# epochTestloss = np.mean(current_step_Test_loss)
# current_step_Test_loss = [] #reset the loss value
# Epoch_Test_losses.append(epochTestloss) # most significant value to store
# epoch_count = epoch_count+1
#
#
# # plt.plot(Epoch_Test_losses)
# # plt.ylabel('loss')
# # plt.xlabel('step')
# # plt.ylim(0, 2)
#
# fig, (ax1, ax2) = plt.subplots(2, 1)
# # ax1 = plt.subplot(2, 1, 1)
# ax1.plot(Epoch_Val_losses)
# ax1.set_ylabel('training loss')
# ax1.set_xlabel('epoch')
# ax1.set_xlim([0, n_epochs])
# ax1.set_ylim([0, 0.4])
# ax1.set_yscale('log')
# # ax1.ylim(0, 0.4)
#
# # ax2 = plt.subplot(2, 1, 2)
# ax2.plot(Epoch_Test_losses)
# ax2.set_ylabel('test loss')
# ax2.set_xlabel('epoch')
# ax2.set_xlim([0, n_epochs])
# ax2.set_ylim([0, 0.1])
# ax2.set_yscale('log')
# # ax2.ylim(0, 0.08)
#
#
# plt.savefig('results/training_epochs_results_reg_{}.png'.format(fileExtension), bbox_inches='tight', pad_inches=0.05)
# # plt.show()
# validation --------------------------------------------------------------------------------------------------------
print('validation phase')
Valcount = 0
processcount = 0
step_Val_loss = []
Epoch_Val_losses = []
from PIL import ImageTk, Image, ImageDraw
epochsValLoss = open("./results/valbothParamShift.txt", "w+")
t = tqdm(iter(val_dataloader), leave=True, total=len(val_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
# image1 = torch.flip(image,[0, 3]) #flip vertical
# image = torch.roll(image, 100, 3) #shift down from 100 px
# image1 = shiftPixel(image, 100, 'y')
# image1 = shiftPixel(image , 100, 'x')
# image1 = torch.flip(image1, [0, 3])
image1 = image
Origimagesave = image1
# Origimagesave = image.to(device)
image1 = image1.to(device) #torch.Size([1, 3, 1024, 1280])
parameter = parameter.to(device)
params = model(image1) # should be size [batchsize, 6]
# print(np.shape(params))
i = 0
# print('image estimated: {}'.format(testcount))
# print('estimated parameter {}'.format(params[i]))
# print('Ground Truth {}'.format(parameter[i]))
epochsValLoss.write('step:{} params:{} \r\n'.format(
processcount,
params.detach().cpu().numpy()))
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
sil2plot = np.squeeze(
(current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
image2show = np.squeeze((Origimagesave[i].detach().cpu().numpy()))
image2show = (image2show * 0.5 + 0.5).transpose(1, 2, 0)
# image2show = np.flip(image2show,1)
# fig = plt.figure()
# fig.add_subplot(2, 1, 1)
# plt.imshow(sil2plot, cmap='gray')
#
# fig.add_subplot(2, 1, 2)
# plt.imshow(image2show)
# plt.show()
sil3d = sil2plot[:, :, np.newaxis]
renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2)
toolIm = Image.fromarray(np.uint8(renderim))
# print(type(image2show))
backgroundIm = Image.fromarray(np.uint8(image2show * 255))
# backgroundIm.show()
#
alpha = 0.2
out = Image.blend(backgroundIm, toolIm, alpha)
# #make gif
imsave('/tmp/_tmp_%04d.png' % processcount, np.array(out))
processcount = processcount + 1
step_Val_loss.append(loss.detach().cpu().numpy())
# print(processcount)
print('making the gif')
make_gif(args.filename_output)
print(step_Val_loss)
print(np.mean(step_Val_loss))
epochsValLoss.close()
def train_renderV3(model, train_dataloader, test_dataloader, n_epochs,
loss_function, date4File, cubeSetName, batch_size,
fileExtension, device, obj_name, noise, number_train_im):
# monitor loss functions as the training progresses
current_step_Train_loss = []
Epoch_Train_losses = []
count = 0
epoch_count = 1
renderCount = 0
regressionCount = 0
lr = 0.001
print('lr used is: {}'.format(lr))
output_result_dir = 'results/render/{}_lr{}'.format(fileExtension, lr)
mkdir_p(output_result_dir)
epochsTrainLoss = open(
"{}/epochsTrainLoss_RenderRegr_{}.txt".format(output_result_dir,
fileExtension), "w+")
TestParamLoss = open(
"{}/TestParamLoss_RenderRegr_{}.txt".format(output_result_dir,
fileExtension), "w+")
x = np.arange(n_epochs)
y = sigmoid(x, 1, 0, n_epochs / 2, 0.2)
plt.plot(x, y)
plt.savefig('{}/ReverseSigmoid_{}.png'.format(output_result_dir,
fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
#usefull for the loss plot of each parameter
epoch_test_loss = []
epoch_test_alpha_loss = []
epoch_test_beta_loss = []
epoch_test_gamma_loss = []
epoch_test_x_loss = []
epoch_test_y_loss = []
epoch_test_z_loss = []
for epoch in range(n_epochs):
## Training phase
model.train()
print('train phase epoch {}/{}'.format(epoch, n_epochs))
alpha = y[
epoch] #proportion of the regression part decrease with negative sigmoid
print('alpha is {}'.format(alpha))
t = tqdm(iter(train_dataloader),
leave=True,
total=len(train_dataloader))
for image, silhouette, parameter in t:
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
# print(params)
numbOfImage = image.size()[0]
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
optimizer.zero_grad()
for i in range(0, numbOfImage):
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R)
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
if (i == 0):
loss = (nn.BCELoss()(current_sil, current_GT_sil).to(
device)) * (1 - alpha) + (nn.MSELoss()(
params[i], parameter[i]).to(device)) * (alpha)
else:
loss += (nn.BCELoss()(current_sil, current_GT_sil).to(
device)) * (1 - alpha) + (nn.MSELoss()(
params[i], parameter[i]).to(device)) * (alpha)
# if (model.t[2] > 0.0317 and model.t[2] < 0.1 and torch.abs(model.t[0]) < 0.06 and torch.abs(model.t[1]) < 0.06):
#
# optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
# optimizer.zero_grad()
# if (i == 0):
# loss = nn.BCELoss()(current_sil, current_GT_sil).to(device)
# else:
# loss = loss + nn.BCELoss()(current_sil, current_GT_sil).to(device)
# print('render')
# renderCount += 1
# else:
# optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
# optimizer.zero_grad()
# if (i == 0):
# loss = nn.MSELoss()(params[i, 3:6], parameter[i, 3:6]).to(device)
# # loss = nn.MSELoss()(params[i], parameter[i]).to(device)
# else:
# loss = loss + nn.MSELoss()(params[i, 3:6], parameter[i, 3:6]).to(device)
# # loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
# print('regression')
# regressionCount += 1
loss = loss / numbOfImage
loss.backward()
optimizer.step()
current_step_Train_loss.append(loss.detach().cpu().numpy(
)) # contain only this epoch loss, will be reset after each epoch
count = count + 1
epochTrainloss = np.mean(current_step_Train_loss)
epochsTrainLoss.write(
'step: {}/{} current step loss: {:.4f}\r\n'.format(
epoch, n_epochs, epochTrainloss))
print('loss of epoch {} is {}'.format(epoch, epochTrainloss))
current_step_Train_loss = [] #reset value
Epoch_Train_losses.append(
epochTrainloss) # most significant value to store
count = 0
testcount = 0
model.eval()
current_step_Test_loss = []
Epoch_Test_losses = []
steps_losses = [] # reset the list after each epoch
steps_alpha_loss = []
steps_beta_loss = []
steps_gamma_loss = []
steps_x_loss = []
steps_y_loss = []
steps_z_loss = []
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:
Test_Step_loss = []
numbOfImage = image.size()[0]
image = image.to(device)
parameter = parameter.to(device)
params = model(image) # should be size [batchsize, 6]
# print(np.shape(params))
for i in range(0, numbOfImage):
print('image tested: {}'.format(testcount))
print('estated {}'.format(params[i]))
print('Ground Truth {}'.format(parameter[i]))
if (i == 0):
loss = nn.MSELoss()(params[i], parameter[i]).to(device)
else:
loss = loss + nn.MSELoss()(params[i],
parameter[i]).to(device)
# one value each for the step, compute mse loss for all parameters separately
alpha_loss = nn.MSELoss()(params[:, 0],
parameter[:,
0]).detach().cpu().numpy()
beta_loss = nn.MSELoss()(params[:, 1],
parameter[:,
1]).detach().cpu().numpy()
gamma_loss = nn.MSELoss()(params[:, 2],
parameter[:,
2]).detach().cpu().numpy()
x_loss = nn.MSELoss()(params[:, 3],
parameter[:, 3]).detach().cpu().numpy()
y_loss = nn.MSELoss()(params[:, 4],
parameter[:, 4]).detach().cpu().numpy()
z_loss = nn.MSELoss()(params[:, 5],
parameter[:, 5]).detach().cpu().numpy()
steps_losses.append(
loss.item()) # only one loss value is add each step
steps_alpha_loss.append(alpha_loss.item())
steps_beta_loss.append(beta_loss.item())
steps_gamma_loss.append(gamma_loss.item())
steps_x_loss.append(x_loss.item())
steps_y_loss.append(y_loss.item())
steps_z_loss.append(z_loss.item())
if epoch_count == n_epochs:
model.t = params[i, 3:6]
R = params[i, 0:3]
model.R = R2Rmat(R) # angle from resnet are in radian
current_sil = model.renderer(model.vertices,
model.faces,
R=model.R,
t=model.t,
mode='silhouettes').squeeze()
current_sil = current_sil[0:1024, 0:1280]
sil2plot = np.squeeze((current_sil.detach().cpu().numpy() *
255)).astype(np.uint8)
current_GT_sil = (silhouette[i] / 255).type(
torch.FloatTensor).to(device)
fig = plt.figure()
fig.add_subplot(2, 1, 1)
plt.imshow(sil2plot, cmap='gray')
fig.add_subplot(2, 1, 2)
plt.imshow(silhouette[i], cmap='gray')
plt.savefig('results/imageRender_{}.png'.format(testcount),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
#MSE loss
current_step_Test_loss.append(loss.detach().cpu().numpy())
testcount = testcount + 1
epoch_test_loss.append(np.mean(steps_losses))
epoch_test_alpha_loss.append(np.mean(steps_alpha_loss))
epoch_test_beta_loss.append(np.mean(steps_beta_loss))
epoch_test_gamma_loss.append(np.mean(steps_gamma_loss))
epoch_test_x_loss.append(np.mean(steps_x_loss))
epoch_test_y_loss.append(np.mean(steps_y_loss))
epoch_test_z_loss.append(np.mean(steps_z_loss))
TestParamLoss.write(
'test epoch: {} current step loss: {:.7f}, angle loss: {:.4f} {:.4f} {:.4f} translation loss: {:.7f} {:.7f} {:.7f}\r\n'
.format(epoch, np.mean(steps_losses), np.mean(steps_alpha_loss),
np.mean(steps_beta_loss), np.mean(steps_x_loss),
np.mean(steps_x_loss), np.mean(steps_z_loss),
np.mean(steps_y_loss)))
epochTestloss = np.mean(current_step_Test_loss)
Epoch_Test_losses.append(
epochTestloss) # most significant value to store
epoch_count = epoch_count + 1
fig, (ax1, ax2) = plt.subplots(2, 1)
# ax1 = plt.subplot(2, 1, 1)
ax1.plot(Epoch_Train_losses)
ax1.set_ylabel('training render BCE loss')
ax1.set_xlabel('epoch')
ax1.set_xlim([0, n_epochs])
ax1.set_ylim([0, 4])
# ax1.set_yscale('log')
# ax2 = plt.subplot(2, 1, 2)
ax2.plot(Epoch_Test_losses)
ax2.set_ylabel('test render MSE loss')
ax2.set_xlabel('epoch')
ax2.set_xlim([0, n_epochs])
ax2.set_ylim([0, 0.1])
# ax2.set_yscale('log')
plt.savefig('{}/training_epochs_rend_results_{}.png'.format(
output_result_dir, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
fig, (a, b, g, x, y, z) = plt.subplots(6, 1)
a.plot(epoch_test_alpha_loss)
a.set_xlim([0, n_epochs])
a.set_ylabel('test alpha loss')
b.plot(epoch_test_beta_loss)
b.set_xlim([0, n_epochs])
b.set_ylabel('test beta loss')
g.plot(epoch_test_gamma_loss)
g.set_xlim([0, n_epochs])
g.set_ylabel('test gamma loss')
x.plot(epoch_test_x_loss)
x.set_xlim([0, n_epochs])
x.set_ylabel('test x loss')
y.plot(epoch_test_y_loss)
y.set_xlim([0, n_epochs])
y.set_ylabel('test y loss')
z.plot(epoch_test_z_loss)
z.set_xlim([0, n_epochs])
z.set_ylabel('test z loss')
z.set_xlabel('epoch')
plt.savefig('{}/test_epochs_rend_Rt_Loss_{}.png'.format(
output_result_dir, fileExtension),
bbox_inches='tight',
pad_inches=0.05)
plt.show()
plt.close()
epochsTrainLoss.close()
TestParamLoss.close()