Example #1
0
def train_renderV2(model, train_dataloader, test_dataloader, n_epochs,
                   loss_function, date4File, cubeSetName, batch_size,
                   fileExtension, device, obj_name, noise, number_train_im):
    # monitor loss functions as the training progresses

    loop = n_epochs
    Step_Val_losses = []
    current_step_loss = []
    current_step_Test_loss = []
    Test_losses = []
    Epoch_Val_losses = []
    Epoch_Test_losses = []
    count = 0
    testcount = 0
    Im2ShowGT = []
    Im2ShowGCP = []
    LastEpochTestCPparam = []
    LastEpochTestGTparam = []
    numbOfImageDataset = number_train_im
    renderCount = 0
    regressionCount = 0
    renderbar = []
    regressionbar = []
    lr = 0.0001  #best at 0.0001 with translation only with span model.t[2] > 4 and model.t[2] < 8 and torch.abs(model.t[0]) < 2 and torch.abs(model.t[1]) < 2):

    output_dir_model = 'models/render/{}'.format(fileExtension)
    mkdir_p(output_dir_model)
    output_dir_results = 'results/render/{}'.format(fileExtension)
    mkdir_p(output_dir_results)

    for epoch in range(n_epochs):

        ## Training phase
        model.train()
        print('train phase epoch {}/{}'.format(epoch, n_epochs))

        t = tqdm(iter(train_dataloader),
                 leave=True,
                 total=len(train_dataloader))
        for image, silhouette, parameter in t:
            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]
            numbOfImage = image.size()[0]
            # loss = nn.MSELoss()(params, parameter).to(device)

            for i in range(0, numbOfImage):
                #create and store silhouette
                model.t = params[i, 3:6]
                R = params[i, 0:3]
                model.R = R2Rmat(R)  # angle from resnet are in radian

                current_sil = model.renderer(model.vertices,
                                             model.faces,
                                             R=model.R,
                                             t=model.t,
                                             mode='silhouettes').squeeze()
                current_GT_sil = (silhouette[i] / 255).type(
                    torch.FloatTensor).to(device)

                if (model.t[2] > 4 and model.t[2] < 8
                        and torch.abs(model.t[0]) < 2
                        and torch.abs(model.t[1]) < 2):
                    # if (epoch > 0):
                    #     optimizer = torch.optim.SGD(model.parameters(), lr=lr)
                    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
                    optimizer.zero_grad()
                    if (i == 0):
                        loss = nn.BCELoss()(current_sil,
                                            current_GT_sil).to(device)
                    else:
                        loss = loss + nn.BCELoss()(current_sil,
                                                   current_GT_sil).to(device)
                    renderCount += 1
                else:
                    # optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
                    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
                    optimizer.zero_grad()
                    if (i == 0):
                        loss = nn.MSELoss()(params[i, 3:6],
                                            parameter[i, 3:6]).to(device)
                        # loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                    else:
                        loss = loss + nn.MSELoss()(
                            params[i, 3:6], parameter[i, 3:6]).to(device)
                        # loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
                    regressionCount += 1

            loss.backward()
            optimizer.step()

            Step_Val_losses.append(loss.detach().cpu().numpy()
                                   )  # contain all step value for all epoch
            current_step_loss.append(loss.detach().cpu().numpy(
            ))  # contain only this epoch loss, will be reset after each epoch
            count = count + 1

#        if (epoch % 5 == 0 and epoch > 2):
#            if (lr > 0.000001):
#                lr = lr / 10
#            print('update lr, is now {}'.format(lr))

        epochValloss = np.mean(current_step_loss)
        current_step_loss = []
        Epoch_Val_losses.append(
            epochValloss)  # most significant value to store
        # print(epochValloss)

        print(Epoch_Val_losses)
        # print(renderCount, regressionCount)

        renderbar.append(renderCount)
        regressionbar.append(regressionCount)
        renderCount = 0
        regressionCount = 0

        torch.save(
            model.state_dict(),
            '{}/{}_{}epoch_{}_Temp_train_{}_{}batchs_{}epochs_Render.pth'.
            format(
                output_dir_model,
                fileExtension,
                epoch,
                date4File,
                cubeSetName,
                str(batch_size),
                str(n_epochs),
            ))
        # validation phase
        print('test phase epoch epoch {}/{}'.format(epoch, n_epochs))
        model.eval()

        t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
        for image, silhouette, parameter in t:

            Test_Step_loss = []
            numbOfImage = image.size()[0]

            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]
            # print(np.shape(params))

            for i in range(0, numbOfImage):
                #create and store silhouette
                model.t = params[i, 3:6]
                R = params[i, 0:3]
                model.R = R2Rmat(R)  # angle from resnet are in radian

                current_sil = model.renderer(model.vertices,
                                             model.faces,
                                             R=model.R,
                                             t=model.t,
                                             mode='silhouettes').squeeze()
                current_GT_sil = (silhouette[i] / 255).type(
                    torch.FloatTensor).to(device)

                if (i == 0):
                    loss = nn.BCELoss()(current_sil, current_GT_sil).to(device)
                else:
                    loss = loss + nn.BCELoss()(current_sil,
                                               current_GT_sil).to(device)

            Test_Step_loss.append(loss.detach().cpu().numpy())

            if (epoch == n_epochs - 1
                ):  # if we are at the last epoch, save param to plot result

                LastEpochTestCPparam.extend(params.detach().cpu().numpy())
                LastEpochTestGTparam.extend(parameter.detach().cpu().numpy())

            Test_losses.append(loss.detach().cpu().numpy())
            current_step_Test_loss.append(loss.detach().cpu().numpy())
            testcount = testcount + 1

        epochTestloss = np.mean(current_step_Test_loss)
        current_step_Test_loss = []
        Epoch_Test_losses.append(
            epochTestloss)  # most significant value to store

# ----------- plot some result from the last epoch computation ------------------------

# print(np.shape(LastEpochTestCPparam)[0])
    nim = 4
    for i in range(0, nim):
        print('saving image to show')
        pickim = int(uniform(0, np.shape(LastEpochTestCPparam)[0] - 1))
        # print(pickim)

        model.t = torch.from_numpy(
            LastEpochTestCPparam[pickim][3:6]).to(device)
        R = torch.from_numpy(LastEpochTestCPparam[pickim][0:3]).to(device)
        model.R = R2Rmat(R)  # angle from resnet are in radia
        imgCP, _, _ = model.renderer(model.vertices,
                                     model.faces,
                                     torch.tanh(model.textures),
                                     R=model.R,
                                     t=model.t)

        model.t = torch.from_numpy(
            LastEpochTestGTparam[pickim][3:6]).to(device)
        R = torch.from_numpy(LastEpochTestGTparam[pickim][0:3]).to(device)
        model.R = R2Rmat(R)  # angle from resnet are in radia
        imgGT, _, _ = model.renderer(model.vertices,
                                     model.faces,
                                     torch.tanh(model.textures),
                                     R=model.R,
                                     t=model.t)

        imgCP = imgCP.squeeze()  # float32 from 0-1
        imgCP = imgCP.detach().cpu().numpy().transpose((1, 2, 0))
        imgCP = (imgCP * 255).astype(
            np.uint8)  # cast from float32 255.0 to 255 uint8
        imgGT = imgGT.squeeze()  # float32 from 0-1
        imgGT = imgGT.detach().cpu().numpy().transpose((1, 2, 0))
        imgGT = (imgGT * 255).astype(
            np.uint8)  # cast from float32 255.0 to 255 uint8
        Im2ShowGT.append(imgCP)
        Im2ShowGCP.append(imgGT)

        a = plt.subplot(2, nim, i + 1)
        plt.imshow(imgGT)
        a.set_title('GT {}'.format(i))
        plt.xticks([0, 512])
        plt.yticks([])
        a = plt.subplot(2, nim, i + 1 + nim)
        plt.imshow(imgCP)
        a.set_title('Rdr {}'.format(i))
        plt.xticks([0, 512])
        plt.yticks([])

    plt.savefig('{}/image_render_{}batch_{}epochs_{}.png'.format(
        output_dir_results, batch_size, n_epochs, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)

    #-----------plot and save section ------------------------------------------------------------------------------------

    fig, (p1, p2, p3, p4) = plt.subplots(4,
                                         figsize=(15, 10))  # largeur hauteur
    fig.suptitle("Render, training {} epochs with {} images, lr={} ".format(
        n_epochs, numbOfImageDataset, lr),
                 fontsize=14)

    moving_aves = RolAv(Step_Val_losses, window=50)
    ind = np.arange(n_epochs)  # index

    p1.plot(np.arange(np.shape(moving_aves)[0]),
            moving_aves,
            label="step Loss rolling average")
    p1.set(ylabel='BCE Step Loss')
    p1.set_yscale('log')
    p1.set(xlabel='Steps')
    p1.set_ylim([0, 20])
    p1.legend()  # Place a legend to the right of this smaller subplot.

    # subplot 2
    p2.plot(np.arange(n_epochs), Epoch_Val_losses, label="Render epoch Loss")
    p2.set(ylabel=' Mean of BCE training step loss')
    p2.set(xlabel='Epochs')
    p2.set_ylim([0, 20])
    p2.set_xticks(ind)
    p2.legend()

    # subplot 3

    width = 0.35
    p3.bar(ind, renderbar, width, color='#d62728', label="render")
    height_cumulative = renderbar
    p3.bar(ind,
           regressionbar,
           width,
           bottom=height_cumulative,
           label="regression")
    p3.set(ylabel='render/regression call')
    p3.set(xlabel='Epochs')
    p3.set_ylim([0, numbOfImageDataset])
    p3.set_xticks(ind)
    p3.legend()

    # subplot 4
    p4.plot(np.arange(n_epochs), Epoch_Test_losses, label="Render Test Loss")
    p4.set(ylabel='Mean of BCE test step loss')
    p4.set(xlabel='Epochs')
    p4.set_ylim([0, 10])
    p4.legend()

    plt.show()

    fig.savefig('{}/render_{}batch_{}epochs_{}.png'.format(
        output_dir_results, batch_size, n_epochs, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    matplotlib2tikz.save("{}/render_{}batch_{}epochs_{}.tex".format(
        output_dir_results, batch_size, n_epochs, fileExtension),
                         figureheight='5.5cm',
                         figurewidth='15cm')

    torch.save(
        model.state_dict(),
        '{}/{}_{}_FinalModel_train_{}_{}batchs_{}epochs_Render.pth'.format(
            output_dir_model,
            fileExtension,
            date4File,
            cubeSetName,
            str(batch_size),
            str(n_epochs),
        ))
    print('parameters saved')
Example #2
0
def Val_V1(model, val_dataloader, n_epochs, fileExtension, device, traintype,
           lr, validation, useofFK, ResnetOutput, SettingString,
           useOwnPretrainedModel):

    output_result_dir = 'results/Validation_{}{}/{}'.format(
        traintype, ResnetOutput, fileExtension)
    mkdir_p(output_result_dir)

    current_dir = os.path.dirname(os.path.realpath(__file__))
    sil_dir = os.path.join(output_result_dir)

    parser = argparse.ArgumentParser()
    parser.add_argument('-or',
                        '--filename_output',
                        type=str,
                        default=os.path.join(
                            sil_dir,
                            'ValidationGif_{}.gif'.format('LongShaft_2')))
    parser.add_argument('-mr', '--make_reference_image', type=int, default=0)
    parser.add_argument('-g', '--gpu', type=int, default=0)
    args = parser.parse_args()

    # validation --------------------------------------------------------------------------------------------------------

    print('validation phase')
    Valcount = 0
    processcount = 0
    step_Val_loss = []
    Epoch_Val_losses = []
    model.eval()
    from PIL import ImageTk, Image, ImageDraw
    paramList = open("./{}/paramList.txt".format(output_result_dir), "w+")

    t = tqdm(iter(val_dataloader), leave=True, total=len(val_dataloader))
    for image, silhouette, parameter in t:

        i = 0
        Test_Step_loss = []
        numbOfImage = image.size()[0]
        # image1 = torch.flip(image,[0, 3]) #flip vertical
        # image = torch.roll(image, 100, 3) #shift down from 100 px
        image1 = shiftPixel(image, 50, 'y')
        image1 = shiftPixel(image1, 50, 'x')
        # image1 = torch.flip(image1, [0, 3])
        # image1 = image
        Origimagesave = image1
        # Origimagesave = image.to(device)
        image1 = image1.to(device)  #torch.Size([1, 3, 1024, 1280])
        parameter = parameter.to(device)

        #parameter estimation through trained model
        if ResnetOutput == 't':  # resnet predict only translation parameter
            print('own model used is t')
            t_params = model(image1)
            model.t = t_params[i]
            R = parameter[
                i,
                0:3]  # give the ground truth parameter for the rotation values
            model.R = R2Rmat(R)
            paramList.write('step:{} params:{} \r\n'.format(
                processcount,
                t_params.detach().cpu().numpy()))
            loss = nn.MSELoss()(t_params[i], parameter[i, 3:6]).to(device)

        if ResnetOutput == 'Rt':  # resnet predict rotation and translation
            params = model(image1)
            print('own model used is Rt')
            model.t = params[i, 3:6]
            R = params[i, 0:3]
            model.R = R2Rmat(R)  # angle from resnet are in radian
            paramList.write('step:{} params:{} \r\n'.format(
                processcount,
                params.detach().cpu().numpy()))
            loss = nn.MSELoss()(params[i], parameter[i]).to(device)

        i = 0
        current_sil = model.renderer(model.vertices,
                                     model.faces,
                                     R=model.R,
                                     t=model.t,
                                     mode='silhouettes').squeeze()
        current_sil = current_sil[0:1024, 0:1280]

        sil2plot = np.squeeze(
            (current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)

        image2show = np.squeeze((Origimagesave[i].detach().cpu().numpy()))
        image2show = (image2show * 0.5 + 0.5).transpose(1, 2, 0)
        # image2show = np.flip(image2show,1)

        # fig = plt.figure()
        # fig.add_subplot(2, 1, 1)
        # plt.imshow(sil2plot, cmap='gray')
        #
        # fig.add_subplot(2, 1, 2)
        # plt.imshow(image2show)
        # plt.show()

        #creation of the blender image
        sil3d = sil2plot[:, :, np.newaxis]
        renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2)
        toolIm = Image.fromarray(np.uint8(renderim))

        # print(type(image2show))
        backgroundIm = Image.fromarray(np.uint8(image2show * 255))

        # backgroundIm.show()
        #

        alpha = 0.2
        out = Image.blend(backgroundIm, toolIm, alpha)
        # #make gif
        imsave('/tmp/_tmp_%04d.png' % processcount, np.array(out))
        processcount = processcount + 1
        step_Val_loss.append(loss.detach().cpu().numpy())
        # print(processcount)

    print('making the gif')
    make_gif(args.filename_output)

    print(step_Val_loss)
    print(np.mean(step_Val_loss))
    paramList.close()
t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
for image, silhouette, parameter in t:

    Test_Step_loss = []
    numbOfImage = image.size()[0]

    image = image.to(device)
    parameter = parameter.to(device)
    params = model(image)  # should be size [batchsize, 6]
    # print(np.shape(params))

    for i in range(0, numbOfImage):
        #create and store silhouette
        model.t = params[i, 3:6]
        R = params[i, 0:3]
        model.R = R2Rmat(R)  # angle from resnet are in radian

        current_sil = model.renderer(model.vertices,
                                     model.faces,
                                     R=model.R,
                                     t=model.t,
                                     mode='silhouettes').squeeze()
        current_GT_sil = (silhouette[i] / 255).type(
            torch.FloatTensor).to(device)

        imgCP, _, _ = model.renderer(model.vertices,
                                     model.faces,
                                     torch.tanh(model.textures),
                                     R=model.R,
                                     t=model.t)
        imgGT = image[i]
Example #4
0
def training(model, train_dataloader, test_dataloader, val_dataloader,
             n_epochs, fileExtension, device, traintype, lr, validation,
             number_test_im, useofFK, ResnetOutput, SettingString,
             useOwnPretrainedModel):
    # monitor loss functions as the training progresses

    current_step_Train_loss = []
    current_step_Train_MSEloss = []

    Epoch_Train_losses = []
    Epoch_Train_MSElosses = []

    count = 0
    epoch_count = 1
    lr = lr
    print('training {}'.format(traintype))
    print('lr used is: {}'.format(lr))

    output_result_dir = 'results/{}{}/{}_lr{}'.format(traintype, ResnetOutput,
                                                      fileExtension, lr)
    mkdir_p(output_result_dir)

    epochsTrainLoss = open(
        "{}/epochsTrainLoss_{}.txt".format(output_result_dir, fileExtension),
        "w+")
    epochsTrainMSELoss = open(
        "{}/epochsTrainMSELoss_{}.txt".format(output_result_dir,
                                              fileExtension), "w+")
    TestParamLoss = open(
        "{}/TestParamLoss_{}.txt".format(output_result_dir, fileExtension),
        "w+")
    ExperimentSettings = open(
        "{}/expSettings_{}.txt".format(output_result_dir, fileExtension), "w+")

    ExperimentSettings.write(SettingString)
    ExperimentSettings.close()

    x = np.arange(n_epochs)
    div = 0.8
    slope = 0.5
    y = sigmoid(x, 1, 0, n_epochs / div, slope)
    plt.plot(x, y)
    # plt.title('div:{} slope:{}'.format(div,slope))
    plt.xlabel('epochs')
    plt.ylabel('\u03B1')
    plt.savefig('{}/ReverseSigmoid_{}.png'.format(output_result_dir,
                                                  fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()

    #usefull for the loss plot of each parameter
    epoch_test_loss = []
    epoch_test_alpha_loss = []
    epoch_test_beta_loss = []
    epoch_test_gamma_loss = []
    epoch_test_x_loss = []
    epoch_test_y_loss = []
    epoch_test_z_loss = []

    Epoch_Test_losses = []

    for epoch in range(n_epochs):

        ## training phase --------------------------------------------------------------------------------------------------------
        model.train()
        print('train phase epoch {}/{}'.format(epoch, n_epochs))

        if useofFK:
            alpha = -1  #y[epoch] #combination of the ground truth with the computed values
        else:
            if useOwnPretrainedModel:
                alpha = 0  #if we continue to train a existing model, we want to train it for pure renderer
            else:
                alpha = y[epoch]

        print('alpha is {}'.format(alpha))

        t = tqdm(iter(train_dataloader),
                 leave=True,
                 total=len(train_dataloader))
        for image, silhouette, parameter in t:
            image = image.to(device)

            if useofFK:
                FKparameters = FKBuild(
                    parameter, np.radians(2), 0.001
                )  #add noise to the ground truth parameter, degree and cm
                FKparameters = FKparameters.to(device)
                params = model(image, FKparameters,
                               useofFK)  # should be size [batchsize, 6]

            else:
                if ResnetOutput == 'Rt':
                    params = model(image)  #call the 6 parameters model
                if ResnetOutput == 't':
                    t_params = model(image)  #call the 3 parameters model
                    # print(t_params.size())

            parameter = parameter.to(device)  #ground truth parameter

            optimizer = torch.optim.Adam(model.parameters(), lr=lr)
            optimizer.zero_grad()

            numbOfImage = image.size()[0]
            mseLoss = 0  #store the mse loss

            for i in range(0, numbOfImage):
                if ResnetOutput == 'Rt':
                    print('Renset output 6 values')
                    model.t = params[i, 3:6]
                    R = params[i, 0:3]
                    model.R = R2Rmat(R)
                    current_sil = model.renderer(model.vertices,
                                                 model.faces,
                                                 R=model.R,
                                                 t=model.t,
                                                 mode='silhouettes').squeeze()
                    current_sil = current_sil[0:1024, 0:1280]
                    current_GT_sil = (silhouette[i] / 255).type(
                        torch.FloatTensor).to(device)

                    if (traintype == 'render'):

                        if useofFK:  #use of the mlp
                            print('render with FK for Rt')
                            if (i == 0):
                                loss = nn.BCELoss()(current_sil,
                                                    current_GT_sil).to(device)
                            else:
                                loss += nn.BCELoss()(current_sil,
                                                     current_GT_sil).to(device)

                        else:  #use sigmoid curve
                            if (i == 0):
                                print('render with sigmoid for Rt')
                                loss = (nn.BCELoss()(current_sil,
                                                     current_GT_sil).to(device)
                                        ) * (1 - alpha) + (nn.MSELoss()(
                                            params[i],
                                            parameter[i]).to(device)) * (alpha)
                            else:
                                loss += (nn.BCELoss()(
                                    current_sil, current_GT_sil).to(
                                        device)) * (1 - alpha) + (nn.MSELoss()(
                                            params[i],
                                            parameter[i]).to(device)) * (alpha)

                    if (traintype == 'regression'):
                        print('regression for Rt')
                        if (i == 0):
                            loss = nn.MSELoss()(params[i],
                                                parameter[i]).to(device)
                        else:
                            loss = loss + nn.MSELoss()(params[i],
                                                       parameter[i]).to(device)
                        print(loss)

                if ResnetOutput == 't':
                    print('Resnet output 3 values')
                    # t_params[i][0]= parameter[i, 4] #overrride x and y
                    # t_params[i][1] = parameter[i, 5]
                    model.t = t_params[i]
                    print(model.t)
                    R = parameter[
                        i, 0:
                        3]  #give the ground truth parameter for the rotation values
                    model.R = R2Rmat(R)
                    current_sil = model.renderer(model.vertices,
                                                 model.faces,
                                                 R=model.R,
                                                 t=model.t,
                                                 mode='silhouettes').squeeze()
                    current_sil = current_sil[0:1024, 0:1280]
                    current_GT_sil = (silhouette[i] / 255).type(
                        torch.FloatTensor).to(device)

                    if (traintype == 'render'):
                        print('t_param is {}'.format(t_params[i]))
                        print('Gt_param is {}'.format(parameter[i, 3:6]))

                        if useofFK:  # use of the mlp
                            print('render with FK for t')
                            if (i == 0):
                                loss = nn.BCELoss()(current_sil,
                                                    current_GT_sil).to(device)
                            else:
                                loss += nn.BCELoss()(current_sil,
                                                     current_GT_sil).to(device)

                        else:  # use sigmoid curve
                            print('render with sigmoid for t')

                            if (i == 0):
                                loss = (nn.BCELoss()(
                                    current_sil, current_GT_sil
                                ).to(device)) * (1 - alpha) + (nn.MSELoss()(
                                    t_params[i],
                                    parameter[i, 3:6]).to(device)) * (alpha)
                                mseLoss = nn.MSELoss()(
                                    t_params[i], parameter[i, 3:6]).to(device)
                            else:
                                loss += (nn.BCELoss()(
                                    current_sil, current_GT_sil
                                ).to(device)) * (1 - alpha) + (nn.MSELoss()(
                                    t_params[i],
                                    parameter[i, 3:6]).to(device)) * (alpha)
                                mseLoss += nn.MSELoss()(
                                    t_params[i], parameter[i, 3:6]).to(device)

                    if (traintype == 'regression'):
                        print('regression for t')
                        if (i == 0):
                            print('t_param is {}'.format(t_params[i]))
                            print('Gt_param is {}'.format(parameter[i, 3:6]))
                            loss = nn.MSELoss()(t_params[i],
                                                parameter[i, 3:6]).to(device)
                        else:
                            loss = loss + nn.MSELoss()(
                                t_params[i], parameter[i, 3:6]).to(device)

            loss = loss / numbOfImage
            mseLoss = mseLoss / numbOfImage
            print('number of image is {}'.format(numbOfImage))
            print('step {} loss is {}'.format(count, loss))
            print('step {} MSE loss is {}'.format(count, mseLoss))
            loss.backward()
            optimizer.step()

            current_step_Train_loss.append(loss.detach().cpu().numpy(
            ))  # contain only this epoch loss, will be reset after each epoch
            current_step_Train_MSEloss.append(mseLoss.detach().cpu().numpy())
            count = count + 1

        epochTrainloss = np.mean(current_step_Train_loss)
        epochTrainMSELoss = np.mean(current_step_Train_MSEloss)
        epochsTrainLoss.write(
            'step: {}/{} current step loss: {:.4f}\r\n'.format(
                epoch, n_epochs, epochTrainloss))
        epochsTrainMSELoss.write(
            'step: {}/{} current step MSE loss: {:.4f}\r\n'.format(
                epoch, n_epochs, epochTrainMSELoss))
        print('loss of epoch {} is {}'.format(epoch, epochTrainloss))
        print('MSE loss of epoch {} is {}'.format(epoch, epochTrainMSELoss))

        # save the model
        output_model_dir = '{}/modelTemp'.format(output_result_dir)
        mkdir_p(output_model_dir)
        torch.save(
            model.state_dict(),
            '{}/TempModel_train_{}{}.pth'.format(output_model_dir,
                                                 fileExtension, count))
        current_step_Train_loss = []  #reset value
        current_step_Train_MSEloss = []
        Epoch_Train_losses.append(
            epochTrainloss)  # most significant value to store
        Epoch_Train_MSElosses.append(epochTrainMSELoss)

        ## test phase --------------------------------------------------------------------------------------------------------
        count = 0
        testcount = 0

        model.eval()

        current_step_Test_loss = []

        steps_losses = []  # reset the list after each epoch
        steps_alpha_loss = []
        steps_beta_loss = []
        steps_gamma_loss = []
        steps_x_loss = []
        steps_y_loss = []
        steps_z_loss = []

        for i in range(number_test_im):
            output_test_image_dir = '{}/images/testIm{}'.format(
                output_result_dir, i)
            mkdir_p(output_test_image_dir)

        t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
        for image, silhouette, parameter in t:

            Test_Step_loss = []
            numbOfImage = image.size()[0]

            image = image.to(device)

            parameter = parameter.to(device)

            if ResnetOutput == 'Rt':
                params = model(image)
            if ResnetOutput == 't':
                t_params = model(image)  # should be
                # print(t_params.size())

            # params = model(image, 0, False)  # should be size [batchsize, 6]
            # print(np.shape(params))

            for i in range(0, numbOfImage):

                if ResnetOutput == 'Rt':
                    print('test for Rt')

                    print('image tested: {}'.format(testcount))
                    print('estimated {}'.format(params[i]))
                    print('Ground Truth {}'.format(parameter[i]))
                    if (i == 0):
                        loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                    else:
                        loss = loss + nn.MSELoss()(params[i],
                                                   parameter[i]).to(device)

                    # one value each for the step, compute mse loss for all parameters separately
                    alpha_loss = nn.MSELoss()(
                        params[:, 0], parameter[:, 0]).detach().cpu().numpy()
                    beta_loss = nn.MSELoss()(
                        params[:, 1], parameter[:, 1]).detach().cpu().numpy()
                    gamma_loss = nn.MSELoss()(
                        params[:, 2], parameter[:, 2]).detach().cpu().numpy()
                    x_loss = nn.MSELoss()(params[:, 3],
                                          parameter[:,
                                                    3]).detach().cpu().numpy()
                    y_loss = nn.MSELoss()(params[:, 4],
                                          parameter[:,
                                                    4]).detach().cpu().numpy()
                    z_loss = nn.MSELoss()(params[:, 5],
                                          parameter[:,
                                                    5]).detach().cpu().numpy()

                    steps_losses.append(
                        loss.item())  # only one loss value is add each step
                    steps_alpha_loss.append(alpha_loss.item())
                    steps_beta_loss.append(beta_loss.item())
                    steps_gamma_loss.append(gamma_loss.item())
                    steps_x_loss.append(x_loss.item())
                    steps_y_loss.append(y_loss.item())
                    steps_z_loss.append(z_loss.item())

                    model.t = params[i, 3:6]
                    R = params[i, 0:3]
                    model.R = R2Rmat(R)  # angle from resnet are in radian

                if ResnetOutput == 't':
                    print('test for t')
                    print('image tested: {}'.format(testcount))
                    print('estimated {}'.format(t_params[i]))
                    print('Ground Truth {}'.format(parameter[i, 3:6]))
                    if (i == 0):
                        loss = nn.MSELoss()(t_params[i],
                                            parameter[i, 3:6]).to(device)
                    else:
                        loss = loss + nn.MSELoss()(
                            t_params[i], parameter[i, 3:6]).to(device)

                    # one value each for the step, compute mse loss for all parameters separately
                    alpha_loss = 0
                    beta_loss = 0
                    gamma_loss = 0
                    x_loss = nn.MSELoss()(t_params[:, 0],
                                          parameter[:,
                                                    3]).detach().cpu().numpy()
                    y_loss = nn.MSELoss()(t_params[:, 1],
                                          parameter[:,
                                                    4]).detach().cpu().numpy()
                    z_loss = nn.MSELoss()(t_params[:, 2],
                                          parameter[:,
                                                    5]).detach().cpu().numpy()

                    steps_losses.append(
                        loss.item())  # only one loss value is add each step
                    steps_alpha_loss.append(0)
                    steps_beta_loss.append(0)
                    steps_gamma_loss.append(0)
                    steps_x_loss.append(x_loss.item())
                    steps_y_loss.append(y_loss.item())
                    steps_z_loss.append(z_loss.item())

                    model.t = t_params[i]
                    R = parameter[
                        i, 0:
                        3]  #give the ground truth parameter for the rotation values
                    model.R = R2Rmat(R)
            print('Rt test GTparameter are {}'.format(parameter))
            print('Rt test parameter are {}{}'.format(R, model.t))
            current_sil = model.renderer(model.vertices,
                                         model.faces,
                                         R=model.R,
                                         t=model.t,
                                         mode='silhouettes').squeeze()
            current_sil = current_sil[0:1024, 0:1280]
            sil2plot = np.squeeze(
                (current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
            current_GT_sil = (silhouette[i] / 255).type(
                torch.FloatTensor).to(device)

            fig = plt.figure()
            fig.add_subplot(2, 1, 1)
            plt.imshow(sil2plot, cmap='gray')

            fig.add_subplot(2, 1, 2)
            plt.imshow(silhouette[i], cmap='gray')
            plt.savefig('{}/images/testIm{}/im{}epoch{}.png'.format(
                output_result_dir, testcount, testcount, epoch),
                        bbox_inches='tight',
                        pad_inches=0.05)
            plt.show()
            plt.close()

            #MSE loss
            current_step_Test_loss.append(loss.detach().cpu().numpy())
            testcount = testcount + 1
    #loop here  for each epoch

        epoch_test_loss.append(np.mean(steps_losses))
        epoch_test_alpha_loss.append(np.mean(steps_alpha_loss))
        epoch_test_beta_loss.append(np.mean(steps_beta_loss))
        epoch_test_gamma_loss.append(np.mean(steps_gamma_loss))
        epoch_test_x_loss.append(np.mean(steps_x_loss))
        epoch_test_y_loss.append(np.mean(steps_y_loss))
        epoch_test_z_loss.append(np.mean(steps_z_loss))

        TestParamLoss.write(
            'test epoch: {} current step loss: {:.7f}, angle loss: {:.4f} {:.4f} {:.4f} translation loss: {:.7f} {:.7f} {:.7f}\r\n'
            .format(epoch, np.mean(steps_losses), np.mean(steps_alpha_loss),
                    np.mean(steps_beta_loss), np.mean(steps_gamma_loss),
                    np.mean(steps_x_loss), np.mean(steps_z_loss),
                    np.mean(steps_y_loss)))

        epochTestloss = np.mean(current_step_Test_loss)
        Epoch_Test_losses.append(
            epochTestloss)  # most significant value to store
        epoch_count = epoch_count + 1

    fig, (ax1, ax11, ax2) = plt.subplots(3, 1)
    ax1.semilogy(Epoch_Train_losses)
    if traintype == 'render':
        if useOwnPretrainedModel:
            ax1.set_ylabel('train {} BCE loss'.format(traintype))
        else:
            ax1.set_ylabel('train {} BCE/MSE loss'.format(traintype))
    else:
        ax1.set_ylabel('train {} MSE loss'.format(traintype))
    ax1.set_xlim([0, n_epochs])
    # ax1.set_ylim([0, 4])
    # ax1.set_yscale('log')

    ax11.semilogy(Epoch_Train_MSElosses)
    ax11.set_ylabel('train {} MSE loss'.format(traintype))
    ax11.set_xlim([0, n_epochs])

    ax2.semilogy(Epoch_Test_losses)
    ax2.set_ylabel('test {} loss'.format(traintype))
    ax2.set_xlabel('epoch')
    ax2.set_xlim([0, n_epochs])
    # ax2.set_ylim([0, 0.1])
    # ax2.set_yscale('log')

    plt.savefig('{}/training_epochs_{}_results_{}.png'.format(
        output_result_dir, traintype, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()
    plt.close()

    fig, (a, b, g) = plt.subplots(3, 1)
    a.semilogy(epoch_test_alpha_loss)
    a.set_xlim([0, n_epochs])
    a.set_ylabel('a')

    b.semilogy(epoch_test_beta_loss)
    b.set_xlim([0, n_epochs])
    b.set_ylabel('b')

    g.semilogy(epoch_test_gamma_loss)
    g.set_xlim([0, n_epochs])
    g.set_ylabel('g')
    g.set_xlabel('MSE loss evolution through epochs')

    plt.savefig('{}/test_epochs_{}_R_Loss_{}.png'.format(
        output_result_dir, traintype, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()
    plt.close()

    fig, (x, y, z) = plt.subplots(3, 1)

    x.semilogy(epoch_test_x_loss)
    x.set_xlim([0, n_epochs])
    x.set_ylabel('x')

    y.semilogy(epoch_test_y_loss)
    y.set_xlim([0, n_epochs])
    y.set_ylabel('y')

    z.semilogy(epoch_test_z_loss)
    z.set_xlim([0, n_epochs])
    z.set_ylabel('z')
    z.set_xlabel('MSE loss evolution through epochs')

    plt.savefig('{}/test_epochs_{}_t_Loss_{}.png'.format(
        output_result_dir, traintype, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()
    plt.close()

    # save the model
    output_model_dir = '{}/model'.format(output_result_dir)
    mkdir_p(output_model_dir)

    torch.save(
        model.state_dict(),
        '{}/FinalModel_train_{}.pth'.format(output_model_dir, fileExtension))
    print('parameters saved')

    epochsTrainLoss.close()
    TestParamLoss.close()
Example #5
0
def train_regressionV2(model, train_dataloader, test_dataloader, n_epochs,
                       loss_function, date4File, cubeSetName, batch_size,
                       fileExtension, device, obj_name, noise,
                       number_train_im):
    # monitor loss functions as the training progresses
    lr = 0.001
    loop = n_epochs
    Step_Val_losses = []
    current_step_loss = []
    current_step_Test_loss = []
    Test_losses = []
    Epoch_Val_losses = []
    Epoch_Test_losses = []
    count = 0
    testcount = 0
    Im2ShowGT = []
    Im2ShowGCP = []
    LastEpochTestCPparam = []
    LastEpochTestGTparam = []
    numbOfImageDataset = number_train_im

    for epoch in range(n_epochs):

        ## Training phase
        model.train()
        print('train phase epoch {}/{}'.format(epoch, n_epochs))
        optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

        t = tqdm(iter(train_dataloader),
                 leave=True,
                 total=len(train_dataloader))
        for image, silhouette, parameter in t:
            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]
            optimizer.zero_grad()
            numbOfImage = image.size()[0]
            # loss = nn.MSELoss()(params, parameter).to(device)

            for i in range(0, numbOfImage):
                #create and store silhouette
                if (i == 0):
                    loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                else:
                    loss = loss + nn.MSELoss()(params[i],
                                               parameter[i]).to(device)

            loss.backward()
            optimizer.step()
            # print(loss)
            Step_Val_losses.append(loss.detach().cpu().numpy()
                                   )  # contain all step value for all epoch
            current_step_loss.append(loss.detach().cpu().numpy(
            ))  # contain only this epoch loss, will be reset after each epoch
            count = count + 1

        epochValloss = np.mean(current_step_loss)
        current_step_loss = []
        Epoch_Val_losses.append(
            epochValloss)  # most significant value to store

        torch.save(
            model.state_dict(),
            'models/{}_{}epoch_{}_TempModel_train_{}_{}batchs_{}epochs_Noise{}_Regression.pth'
            .format(
                fileExtension,
                epoch,
                date4File,
                cubeSetName,
                str(batch_size),
                str(n_epochs),
                noise * 100,
            ))
        # print(epochValloss)

        # validation phase
        print('test phase epoch epoch {}/{}'.format(epoch, n_epochs))
        model.eval()

        t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
        for image, silhouette, parameter in t:

            Test_Step_loss = []
            numbOfImage = image.size()[0]

            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]
            # print(np.shape(params))

            for i in range(0, numbOfImage):
                if (i == 0):
                    loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                else:
                    loss = loss + nn.MSELoss()(params[i],
                                               parameter[i]).to(device)

            Test_Step_loss.append(loss.detach().cpu().numpy())

            if (epoch == n_epochs - 1
                ):  # if we are at the last epoch, save param to plot result

                LastEpochTestCPparam.extend(params.detach().cpu().numpy())
                LastEpochTestGTparam.extend(parameter.detach().cpu().numpy())

            Test_losses.append(loss.detach().cpu().numpy())
            current_step_Test_loss.append(loss.detach().cpu().numpy())
            testcount = testcount + 1

        epochTestloss = np.mean(current_step_Test_loss)
        current_step_Test_loss = []
        Epoch_Test_losses.append(
            epochTestloss)  # most significant value to store

    # ----------- plot some result from the last epoch computation ------------------------

    # print(np.shape(LastEpochTestCPparam)[0])
    nim = 5
    for i in range(0, nim):
        print('saving image to show')
        pickim = int(uniform(0, np.shape(LastEpochTestCPparam)[0] - 1))
        # print(pickim)

        model.t = torch.from_numpy(
            LastEpochTestCPparam[pickim][3:6]).to(device)
        R = torch.from_numpy(LastEpochTestCPparam[pickim][0:3]).to(device)
        model.R = R2Rmat(R)  # angle from resnet are in radia
        imgCP, _, _ = model.renderer(model.vertices,
                                     model.faces,
                                     torch.tanh(model.textures),
                                     R=model.R,
                                     t=model.t)

        model.t = torch.from_numpy(
            LastEpochTestGTparam[pickim][3:6]).to(device)
        R = torch.from_numpy(LastEpochTestGTparam[pickim][0:3]).to(device)
        model.R = R2Rmat(R)  # angle from resnet are in radia
        imgGT, _, _ = model.renderer(model.vertices,
                                     model.faces,
                                     torch.tanh(model.textures),
                                     R=model.R,
                                     t=model.t)

        imgCP = imgCP.squeeze()  # float32 from 0-1
        imgCP = imgCP.detach().cpu().numpy().transpose((1, 2, 0))
        imgCP = (imgCP * 255).astype(
            np.uint8)  # cast from float32 255.0 to 255 uint8
        imgGT = imgGT.squeeze()  # float32 from 0-1
        imgGT = imgGT.detach().cpu().numpy().transpose((1, 2, 0))
        imgGT = (imgGT * 255).astype(
            np.uint8)  # cast from float32 255.0 to 255 uint8
        Im2ShowGT.append(imgCP)
        Im2ShowGCP.append(imgGT)

        a = plt.subplot(2, nim, i + 1)
        plt.imshow(imgGT)
        a.set_title('GT {}'.format(i))
        plt.xticks([0, 512])
        plt.yticks([])
        a = plt.subplot(2, nim, i + 1 + nim)
        plt.imshow(imgCP)
        a.set_title('Rdr {}'.format(i))
        plt.xticks([0, 512])
        plt.yticks([])

    plt.savefig('results/image_regression_{}batch_{}_{}.pdf'.format(
        batch_size, n_epochs, fileExtension))

    # -----------plot and save section ------------------------------------------------------------------------------------

    fig, (p1, p2, p4) = plt.subplots(3, figsize=(15, 10))  # largeur hauteur

    moving_aves = RolAv(Step_Val_losses, window=20)

    p1.plot(np.arange(np.shape(moving_aves)[0]),
            moving_aves,
            label="step Loss rolling average")
    p1.set(ylabel='BCE Step Loss')
    p1.set_yscale('log')
    p1.set(xlabel='Steps')
    p1.set_ylim([0, 4])
    p1.legend()  # Place a legend to the right of this smaller subplot.

    # subplot 2
    p2.plot(np.arange(n_epochs), Epoch_Val_losses, label="epoch Loss")
    p2.set(ylabel=' Mean of BCE training step loss')
    p2.set(xlabel='Epochs')
    p2.set_ylim([0, 4])
    p2.legend()

    p4.plot(np.arange(n_epochs), Epoch_Test_losses, label="Test Loss")
    p4.set(ylabel='Mean of BCE test step loss')
    p4.set(xlabel='Epochs')
    p4.set_ylim([0, 2])
    p4.legend()

    plt.show()

    fig.savefig('results/regression_{}batch_{}_{}.pdf'.format(
        batch_size, n_epochs, fileExtension))
    import matplotlib2tikz

    matplotlib2tikz.save("results/regression_{}batch_{}_{}.tex".format(
        batch_size, n_epochs, fileExtension))
def train_regV3(model, train_dataloader, test_dataloader, n_epochs,
                loss_function, date4File, cubeSetName, batch_size,
                fileExtension, device, obj_name, noise, number_train_im,
                val_dataloader):
    # monitor loss functions as the training progresses

    loop = n_epochs
    Step_Val_losses = []
    current_step_loss = []
    current_step_Test_loss = []
    Test_losses = []
    Epoch_Val_losses = []
    allstepvalloss = []
    Epoch_Test_losses = []
    count = 0
    epoch_count = 1
    testcount = 0
    Im2ShowGT = []
    Im2ShowGCP = []
    LastEpochTestCPparam = []
    LastEpochTestGTparam = []
    numbOfImageDataset = number_train_im
    renderCount = 0
    regressionCount = 0
    renderbar = []
    regressionbar = []
    lr = 0.00001

    # training -------------------------------------------------------------------------------------------------------

    # for epoch in range(n_epochs):
    #
    #     ## Training phase
    #     model.train()
    #     print('train phase epoch {}/{}'.format(epoch, n_epochs))
    #     optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
    #
    #     t = tqdm(iter(train_dataloader), leave=True, total=len(train_dataloader))
    #     for image, silhouette, parameter in t:
    #         image = image.to(device)
    #         parameter = parameter.to(device)
    #         params = model(image)  # should be size [batchsize, 6]
    #         optimizer.zero_grad()
    #
    #         # print(params)
    #         numbOfImage = image.size()[0]
    #         # loss = nn.MSELoss()(params, parameter).to(device)
    #
    #
    #         for i in range(0,numbOfImage):
    #             #create and store silhouette
    #             if (i == 0):
    #                 loss =nn.MSELoss()(params[i], parameter[i]).to(device)
    #             else:
    #                 loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
    #
    #
    #         loss.backward()
    #         optimizer.step()
    #
    #         print(loss)
    #         current_step_loss.append(loss.detach().cpu().numpy())  # contain only this epoch loss, will be reset after each epoch
    #         count = count + 1
    #
    #     epochValloss = np.mean(current_step_loss)
    #     current_step_loss = [] #reset value
    #     Epoch_Val_losses.append(epochValloss)  # most significant value to store
    #     # print(epochValloss)
    #
    #     print(Epoch_Val_losses)
    #
    #
    #     #test --------------------------------------------------------------------------------------------------------
    #
    #     count = 0
    #     testcount = 0
    #     model.eval()
    #
    #     t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
    #     for image, silhouette, parameter in t:
    #
    #         Test_Step_loss = []
    #         numbOfImage = image.size()[0]
    #
    #         image = image.to(device)
    #         parameter = parameter.to(device)
    #         params = model(image)  # should be size [batchsize, 6]
    #         # print(np.shape(params))
    #
    #         for i in range(0, numbOfImage):
    #             print('image tested: {}'.format(testcount))
    #             print('estated {}'.format(params[i]))
    #             print('Ground Truth {}'.format(parameter[i]))
    #             if (i == 0):
    #                 loss = nn.MSELoss()(params[i], parameter[i]).to(device)
    #             else:
    #                 loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
    #
    #         if epoch_count == n_epochs:
    #             model.t = params[i, 3:6]
    #             R = params[i, 0:3]
    #             model.R = R2Rmat(R)  # angle from resnet are in radian
    #
    #             current_sil = model.renderer(model.vertices, model.faces, R=model.R, t=model.t,
    #                                          mode='silhouettes').squeeze()
    #             current_sil = current_sil[0:1024, 0:1280]
    #             sil2plot = np.squeeze((current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)
    #             current_GT_sil = (silhouette[i] / 255).type(torch.FloatTensor).to(device)
    #
    #             fig = plt.figure()
    #             fig.add_subplot(2, 1, 1)
    #             plt.imshow(sil2plot, cmap='gray')
    #
    #             fig.add_subplot(2, 1, 2)
    #             plt.imshow(silhouette[i], cmap='gray')
    #             plt.savefig('results/image_{}.png'.format(testcount), bbox_inches='tight',
    #                         pad_inches=0.05)
    #             # plt.show()
    #
    #         current_step_Test_loss.append(loss.detach().cpu().numpy())
    #         testcount = testcount + 1
    #
    #     epochTestloss = np.mean(current_step_Test_loss)
    #     current_step_Test_loss = [] #reset the loss value
    #     Epoch_Test_losses.append(epochTestloss)  # most significant value to store
    #     epoch_count = epoch_count+1
    #
    #
    # # plt.plot(Epoch_Test_losses)
    # # plt.ylabel('loss')
    # # plt.xlabel('step')
    # # plt.ylim(0, 2)
    #
    # fig, (ax1, ax2) = plt.subplots(2, 1)
    # # ax1 = plt.subplot(2, 1, 1)
    # ax1.plot(Epoch_Val_losses)
    # ax1.set_ylabel('training loss')
    # ax1.set_xlabel('epoch')
    # ax1.set_xlim([0, n_epochs])
    # ax1.set_ylim([0, 0.4])
    # ax1.set_yscale('log')
    # # ax1.ylim(0, 0.4)
    #
    # # ax2 = plt.subplot(2, 1, 2)
    # ax2.plot(Epoch_Test_losses)
    # ax2.set_ylabel('test loss')
    # ax2.set_xlabel('epoch')
    # ax2.set_xlim([0, n_epochs])
    # ax2.set_ylim([0, 0.1])
    # ax2.set_yscale('log')
    # # ax2.ylim(0, 0.08)
    #
    #
    # plt.savefig('results/training_epochs_results_reg_{}.png'.format(fileExtension), bbox_inches='tight', pad_inches=0.05)
    # # plt.show()

    # validation --------------------------------------------------------------------------------------------------------

    print('validation phase')
    Valcount = 0
    processcount = 0
    step_Val_loss = []
    Epoch_Val_losses = []
    from PIL import ImageTk, Image, ImageDraw
    epochsValLoss = open("./results/valbothParamShift.txt", "w+")

    t = tqdm(iter(val_dataloader), leave=True, total=len(val_dataloader))
    for image, silhouette, parameter in t:

        Test_Step_loss = []
        numbOfImage = image.size()[0]
        # image1 = torch.flip(image,[0, 3]) #flip vertical
        # image = torch.roll(image, 100, 3) #shift down from 100 px
        # image1 = shiftPixel(image, 100, 'y')
        # image1 =   shiftPixel(image , 100, 'x')
        # image1 = torch.flip(image1, [0, 3])
        image1 = image
        Origimagesave = image1
        # Origimagesave = image.to(device)
        image1 = image1.to(device)  #torch.Size([1, 3, 1024, 1280])
        parameter = parameter.to(device)
        params = model(image1)  # should be size [batchsize, 6]
        # print(np.shape(params))
        i = 0

        # print('image estimated: {}'.format(testcount))
        # print('estimated parameter {}'.format(params[i]))
        # print('Ground Truth {}'.format(parameter[i]))

        epochsValLoss.write('step:{} params:{} \r\n'.format(
            processcount,
            params.detach().cpu().numpy()))
        loss = nn.MSELoss()(params[i], parameter[i]).to(device)

        model.t = params[i, 3:6]
        R = params[i, 0:3]
        model.R = R2Rmat(R)  # angle from resnet are in radian

        current_sil = model.renderer(model.vertices,
                                     model.faces,
                                     R=model.R,
                                     t=model.t,
                                     mode='silhouettes').squeeze()
        current_sil = current_sil[0:1024, 0:1280]

        sil2plot = np.squeeze(
            (current_sil.detach().cpu().numpy() * 255)).astype(np.uint8)

        image2show = np.squeeze((Origimagesave[i].detach().cpu().numpy()))
        image2show = (image2show * 0.5 + 0.5).transpose(1, 2, 0)
        # image2show = np.flip(image2show,1)

        # fig = plt.figure()
        # fig.add_subplot(2, 1, 1)
        # plt.imshow(sil2plot, cmap='gray')
        #
        # fig.add_subplot(2, 1, 2)
        # plt.imshow(image2show)
        # plt.show()

        sil3d = sil2plot[:, :, np.newaxis]
        renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2)
        toolIm = Image.fromarray(np.uint8(renderim))

        # print(type(image2show))
        backgroundIm = Image.fromarray(np.uint8(image2show * 255))

        # backgroundIm.show()
        #

        alpha = 0.2
        out = Image.blend(backgroundIm, toolIm, alpha)
        # #make gif
        imsave('/tmp/_tmp_%04d.png' % processcount, np.array(out))
        processcount = processcount + 1
        step_Val_loss.append(loss.detach().cpu().numpy())
        # print(processcount)

    print('making the gif')
    make_gif(args.filename_output)

    print(step_Val_loss)
    print(np.mean(step_Val_loss))
    epochsValLoss.close()
Example #7
0
def train_renderV3(model, train_dataloader, test_dataloader, n_epochs,
                   loss_function, date4File, cubeSetName, batch_size,
                   fileExtension, device, obj_name, noise, number_train_im):
    # monitor loss functions as the training progresses

    current_step_Train_loss = []
    Epoch_Train_losses = []

    count = 0
    epoch_count = 1
    renderCount = 0
    regressionCount = 0
    lr = 0.001
    print('lr used is: {}'.format(lr))

    output_result_dir = 'results/render/{}_lr{}'.format(fileExtension, lr)
    mkdir_p(output_result_dir)

    epochsTrainLoss = open(
        "{}/epochsTrainLoss_RenderRegr_{}.txt".format(output_result_dir,
                                                      fileExtension), "w+")
    TestParamLoss = open(
        "{}/TestParamLoss_RenderRegr_{}.txt".format(output_result_dir,
                                                    fileExtension), "w+")

    x = np.arange(n_epochs)
    y = sigmoid(x, 1, 0, n_epochs / 2, 0.2)
    plt.plot(x, y)
    plt.savefig('{}/ReverseSigmoid_{}.png'.format(output_result_dir,
                                                  fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()

    #usefull for the loss plot of each parameter
    epoch_test_loss = []
    epoch_test_alpha_loss = []
    epoch_test_beta_loss = []
    epoch_test_gamma_loss = []
    epoch_test_x_loss = []
    epoch_test_y_loss = []
    epoch_test_z_loss = []

    for epoch in range(n_epochs):

        ## Training phase
        model.train()
        print('train phase epoch {}/{}'.format(epoch, n_epochs))

        alpha = y[
            epoch]  #proportion of the regression part decrease with negative sigmoid

        print('alpha is {}'.format(alpha))

        t = tqdm(iter(train_dataloader),
                 leave=True,
                 total=len(train_dataloader))
        for image, silhouette, parameter in t:
            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]

            # print(params)
            numbOfImage = image.size()[0]
            optimizer = torch.optim.Adam(model.parameters(), lr=lr)
            optimizer.zero_grad()

            for i in range(0, numbOfImage):

                model.t = params[i, 3:6]
                R = params[i, 0:3]
                model.R = R2Rmat(R)
                current_sil = model.renderer(model.vertices,
                                             model.faces,
                                             R=model.R,
                                             t=model.t,
                                             mode='silhouettes').squeeze()
                current_sil = current_sil[0:1024, 0:1280]
                current_GT_sil = (silhouette[i] / 255).type(
                    torch.FloatTensor).to(device)

                if (i == 0):
                    loss = (nn.BCELoss()(current_sil, current_GT_sil).to(
                        device)) * (1 - alpha) + (nn.MSELoss()(
                            params[i], parameter[i]).to(device)) * (alpha)
                else:
                    loss += (nn.BCELoss()(current_sil, current_GT_sil).to(
                        device)) * (1 - alpha) + (nn.MSELoss()(
                            params[i], parameter[i]).to(device)) * (alpha)

                # if (model.t[2] > 0.0317 and model.t[2] < 0.1 and torch.abs(model.t[0]) < 0.06 and torch.abs(model.t[1]) < 0.06):
                #
                #     optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
                #     optimizer.zero_grad()
                #     if (i == 0):
                #         loss  =  nn.BCELoss()(current_sil, current_GT_sil).to(device)
                #     else:
                #         loss = loss + nn.BCELoss()(current_sil, current_GT_sil).to(device)
                #     print('render')
                #     renderCount += 1
                # else:
                #     optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
                #     optimizer.zero_grad()
                #     if (i == 0):
                #         loss = nn.MSELoss()(params[i, 3:6], parameter[i, 3:6]).to(device)
                #         # loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                #     else:
                #         loss = loss + nn.MSELoss()(params[i, 3:6], parameter[i, 3:6]).to(device)
                #         # loss = loss + nn.MSELoss()(params[i], parameter[i]).to(device)
                #     print('regression')
                #     regressionCount += 1

            loss = loss / numbOfImage
            loss.backward()
            optimizer.step()

            current_step_Train_loss.append(loss.detach().cpu().numpy(
            ))  # contain only this epoch loss, will be reset after each epoch
            count = count + 1

        epochTrainloss = np.mean(current_step_Train_loss)
        epochsTrainLoss.write(
            'step: {}/{} current step loss: {:.4f}\r\n'.format(
                epoch, n_epochs, epochTrainloss))
        print('loss of epoch {} is {}'.format(epoch, epochTrainloss))
        current_step_Train_loss = []  #reset value
        Epoch_Train_losses.append(
            epochTrainloss)  # most significant value to store

        count = 0
        testcount = 0

        model.eval()

        current_step_Test_loss = []
        Epoch_Test_losses = []
        steps_losses = []  # reset the list after each epoch
        steps_alpha_loss = []
        steps_beta_loss = []
        steps_gamma_loss = []
        steps_x_loss = []
        steps_y_loss = []
        steps_z_loss = []

        t = tqdm(iter(test_dataloader), leave=True, total=len(test_dataloader))
        for image, silhouette, parameter in t:

            Test_Step_loss = []
            numbOfImage = image.size()[0]

            image = image.to(device)
            parameter = parameter.to(device)
            params = model(image)  # should be size [batchsize, 6]
            # print(np.shape(params))

            for i in range(0, numbOfImage):
                print('image tested: {}'.format(testcount))
                print('estated {}'.format(params[i]))
                print('Ground Truth {}'.format(parameter[i]))
                if (i == 0):
                    loss = nn.MSELoss()(params[i], parameter[i]).to(device)
                else:
                    loss = loss + nn.MSELoss()(params[i],
                                               parameter[i]).to(device)

                # one value each for the step, compute mse loss for all parameters separately
                alpha_loss = nn.MSELoss()(params[:, 0],
                                          parameter[:,
                                                    0]).detach().cpu().numpy()
                beta_loss = nn.MSELoss()(params[:, 1],
                                         parameter[:,
                                                   1]).detach().cpu().numpy()
                gamma_loss = nn.MSELoss()(params[:, 2],
                                          parameter[:,
                                                    2]).detach().cpu().numpy()
                x_loss = nn.MSELoss()(params[:, 3],
                                      parameter[:, 3]).detach().cpu().numpy()
                y_loss = nn.MSELoss()(params[:, 4],
                                      parameter[:, 4]).detach().cpu().numpy()
                z_loss = nn.MSELoss()(params[:, 5],
                                      parameter[:, 5]).detach().cpu().numpy()

                steps_losses.append(
                    loss.item())  # only one loss value is add each step
                steps_alpha_loss.append(alpha_loss.item())
                steps_beta_loss.append(beta_loss.item())
                steps_gamma_loss.append(gamma_loss.item())
                steps_x_loss.append(x_loss.item())
                steps_y_loss.append(y_loss.item())
                steps_z_loss.append(z_loss.item())

            if epoch_count == n_epochs:
                model.t = params[i, 3:6]
                R = params[i, 0:3]
                model.R = R2Rmat(R)  # angle from resnet are in radian

                current_sil = model.renderer(model.vertices,
                                             model.faces,
                                             R=model.R,
                                             t=model.t,
                                             mode='silhouettes').squeeze()
                current_sil = current_sil[0:1024, 0:1280]
                sil2plot = np.squeeze((current_sil.detach().cpu().numpy() *
                                       255)).astype(np.uint8)
                current_GT_sil = (silhouette[i] / 255).type(
                    torch.FloatTensor).to(device)

                fig = plt.figure()
                fig.add_subplot(2, 1, 1)
                plt.imshow(sil2plot, cmap='gray')

                fig.add_subplot(2, 1, 2)
                plt.imshow(silhouette[i], cmap='gray')
                plt.savefig('results/imageRender_{}.png'.format(testcount),
                            bbox_inches='tight',
                            pad_inches=0.05)
                plt.show()

            #MSE loss
            current_step_Test_loss.append(loss.detach().cpu().numpy())
            testcount = testcount + 1

        epoch_test_loss.append(np.mean(steps_losses))
        epoch_test_alpha_loss.append(np.mean(steps_alpha_loss))
        epoch_test_beta_loss.append(np.mean(steps_beta_loss))
        epoch_test_gamma_loss.append(np.mean(steps_gamma_loss))
        epoch_test_x_loss.append(np.mean(steps_x_loss))
        epoch_test_y_loss.append(np.mean(steps_y_loss))
        epoch_test_z_loss.append(np.mean(steps_z_loss))

        TestParamLoss.write(
            'test epoch: {} current step loss: {:.7f}, angle loss: {:.4f} {:.4f} {:.4f} translation loss: {:.7f} {:.7f} {:.7f}\r\n'
            .format(epoch, np.mean(steps_losses), np.mean(steps_alpha_loss),
                    np.mean(steps_beta_loss), np.mean(steps_x_loss),
                    np.mean(steps_x_loss), np.mean(steps_z_loss),
                    np.mean(steps_y_loss)))

        epochTestloss = np.mean(current_step_Test_loss)
        Epoch_Test_losses.append(
            epochTestloss)  # most significant value to store
        epoch_count = epoch_count + 1

    fig, (ax1, ax2) = plt.subplots(2, 1)
    # ax1 = plt.subplot(2, 1, 1)
    ax1.plot(Epoch_Train_losses)
    ax1.set_ylabel('training render BCE loss')
    ax1.set_xlabel('epoch')
    ax1.set_xlim([0, n_epochs])
    ax1.set_ylim([0, 4])
    # ax1.set_yscale('log')

    # ax2 = plt.subplot(2, 1, 2)
    ax2.plot(Epoch_Test_losses)
    ax2.set_ylabel('test render MSE loss')
    ax2.set_xlabel('epoch')
    ax2.set_xlim([0, n_epochs])
    ax2.set_ylim([0, 0.1])
    # ax2.set_yscale('log')

    plt.savefig('{}/training_epochs_rend_results_{}.png'.format(
        output_result_dir, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()
    plt.close()

    fig, (a, b, g, x, y, z) = plt.subplots(6, 1)
    a.plot(epoch_test_alpha_loss)
    a.set_xlim([0, n_epochs])
    a.set_ylabel('test alpha loss')

    b.plot(epoch_test_beta_loss)
    b.set_xlim([0, n_epochs])
    b.set_ylabel('test beta loss')

    g.plot(epoch_test_gamma_loss)
    g.set_xlim([0, n_epochs])
    g.set_ylabel('test gamma loss')

    x.plot(epoch_test_x_loss)
    x.set_xlim([0, n_epochs])
    x.set_ylabel('test x loss')

    y.plot(epoch_test_y_loss)
    y.set_xlim([0, n_epochs])
    y.set_ylabel('test y loss')

    z.plot(epoch_test_z_loss)
    z.set_xlim([0, n_epochs])
    z.set_ylabel('test z loss')
    z.set_xlabel('epoch')

    plt.savefig('{}/test_epochs_rend_Rt_Loss_{}.png'.format(
        output_result_dir, fileExtension),
                bbox_inches='tight',
                pad_inches=0.05)
    plt.show()
    plt.close()

    epochsTrainLoss.close()
    TestParamLoss.close()