imageFGfake = tf.placeholder(tf.float32,shape=[opt.batchSize,opt.H,opt.W,4]) PH = [imageBGfakeData,imageRealData,imageFGfake] # ------ generate perturbation ------ imageReal = data.perturbBG(opt,imageRealData) imageBGfake = data.perturbBG(opt,imageBGfakeData) pPertFG = opt.pertFG*tf.random_normal([opt.batchSize,opt.warpDim]) # ------ define GP and D ------ geometric = graph.geometric_multires discriminator = graph.discriminator # ------ geometric predictor ------ imageFGwarpAll,pAll,_ = geometric(opt,imageBGfake,imageFGfake,pPertFG) pWarp = pAll[-1] # ------ composite image ------ summaryImageTrain = [] summaryImageTest = [] summaryImageTrain.append(util.imageSummary(opt,imageReal,"TRAIN_real",opt.H,opt.W)) summaryImageTest.append(util.imageSummary(opt,imageReal,"TEST_real",opt.H,opt.W)) imageFGwarp = imageFGwarpAll[0] imageComp = graph.composite(opt,imageBGfake,imageFGwarp) summaryImageTrain.append(util.imageSummary(opt,imageComp,"TRAIN_compST{0}".format(0),opt.H,opt.W)) summaryImageTest.append(util.imageSummary(opt,imageComp,"TEST_compST{0}".format(0),opt.H,opt.W)) alpha = tf.random_uniform(shape=[opt.batchSize,1,1,1]) imageIntp = alpha*imageReal+(1-alpha)*imageComp # ------ discriminator ------ outComps,outIntps = [],[] outReal = discriminator(opt,imageReal) outComp = discriminator(opt,imageComp,reuse=True) outIntp = discriminator(opt,imageIntp,reuse=True) # ------ discriminator gradient ------ grad_D_fake = tf.gradients(outIntp,imageIntp)[0] grad_D_norm = tf.sqrt(tf.reduce_sum(grad_D_fake**2+1e-8,reduction_indices=[1,2,3]))
fuseTrans = tf.nn.l2_normalize(opt.fuseTrans, dim=1) XYZid, ML = transform.fuse3D(opt, XYZ, maskLogit, fuseTrans) # [B,1,VHW] newDepth, newMaskLogit, collision = transform.render2D( opt, XYZid, ML, renderTrans) # [B,N,H,W,1] # ------ define loss ------ loss_depth = graph.masked_l1_loss(newDepth - depthGT, tf.equal( collision, 1)) / (opt.batchSize * opt.novelN) loss_mask = graph.cross_entropy_loss(newMaskLogit, maskGT) / (opt.batchSize * opt.novelN) loss = loss_mask + opt.lambdaDepth * loss_depth # ------ optimizer ------ lr_PH = tf.placeholder(tf.float32, shape=[]) optim = tf.train.AdamOptimizer(learning_rate=lr_PH).minimize(loss) # ------ generate summaries ------ summaryImage = [ util.imageSummary(opt, "image_RGB", inputImage, opt.inH, opt.inW), util.imageSummary(opt, "image_depth/pred", ((1 - newDepth) * tf.to_float(tf.equal(collision, 1)))[:, 0, :, :, 0:1], opt.H, opt.W), util.imageSummary(opt, "image_depth/GT", (1 - depthGT)[:, 0, :, :, 0:1], opt.H, opt.W), util.imageSummary(opt, "image_mask/new", tf.sigmoid(newMaskLogit[:, 0, :, :, 0:1]), opt.H, opt.W), util.imageSummary(opt, "image_mask", tf.sigmoid(maskLogit[:, :, :, 0:1]), opt.outH, opt.outW), util.imageSummary(opt, "image_mask/GT", maskGT[:, 0, :, :, 0:1], opt.H, opt.W)
PH = [imageBGfakeData, imageRealData, imageFGfake] # ------ generate perturbation ------ imageReal = imageRealData imageBGfake = imageBGfakeData pPertFG = opt.pertFG * tf.random_normal([opt.batchSize, opt.warpDim]) # ------ define GP and D ------ geometric = graph.geometric_multires discriminator = graph.discriminator # ------ geometric predictor ------ imageFGwarpAll, pAll, _ = geometric(opt, imageBGfake, imageFGfake, pPertFG) pWarp = pAll[-1] # ------ composite image ------ summaryImageTrain = [] summaryImageTest = [] summaryImageTrain.append( util.imageSummary(opt, imageReal, "TRAIN_real", opt.H, opt.W)) summaryImageTest.append( util.imageSummary(opt, imageReal, "TEST_real", opt.H, opt.W)) imageFGwarp = imageFGwarpAll[0] imageComp = graph.composite(opt, imageBGfake, imageFGwarp) summaryImageTrain.append( util.imageSummary(opt, imageComp, "TRAIN_compST{0}".format(0), opt.H, opt.W)) summaryImageTest.append( util.imageSummary(opt, imageComp, "TEST_compST{0}".format(0), opt.H, opt.W)) alpha = tf.random_uniform(shape=[opt.batchSize, 1, 1, 1]) imageIntp = alpha * imageReal + (1 - alpha) * imageComp # ------ discriminator ------ outComps, outIntps = [], [] outReal = discriminator(opt, imageReal)
], axis=0) # [V,H,W] XYGT = np.expand_dims(np.transpose(XYGT, axes=[1, 2, 0]), axis=0) # [1,H,W,2V] XY = XYZ[:, :, :, :opt.outViewN * 2] loss_XYZ = graph.l1_loss(XY - XYGT) / opt.batchSize loss_XYZ += graph.masked_l1_loss(depth - depthGT, maskLogit > 0) / opt.batchSize loss_mask = graph.cross_entropy_loss(maskLogit, maskGT) / opt.batchSize loss = loss_mask + opt.lambdaDepth * loss_XYZ # ------ optimizer ------ lr_PH = tf.placeholder(tf.float32, shape=[]) optim = tf.train.AdamOptimizer(learning_rate=lr_PH).minimize(loss) # ------ generate summaries ------ summaryImage = [ util.imageSummary(opt, "image_RGB", inputImage, opt.inH, opt.inW), util.imageSummary(opt, "image_depth/pred", (1 - depth)[:, :, :, 0:1], opt.outH, opt.outW), util.imageSummary(opt, "image_depth/valid", ((1 - depth) * mask)[:, :, :, 0:1], opt.outH, opt.outW), util.imageSummary(opt, "image_depth/GT", (1 - depthGT)[:, :, :, 0:1], opt.outH, opt.outW), util.imageSummary(opt, "image_mask", tf.sigmoid(maskLogit[:, :, :, 0:1]), opt.outH, opt.outW), util.imageSummary(opt, "image_mask/GT", maskGT[:, :, :, 0:1], opt.outH, opt.outW) ] summaryImage = tf.summary.merge(summaryImage) summaryLoss = [
output = graph.CNN(opt,imageWarp) softmax = tf.nn.softmax(output) labelOnehot = tf.one_hot(label,opt.labelN) prediction = tf.equal(tf.argmax(softmax,1),label) # ------ define loss ------ softmaxLoss = tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labelOnehot) loss = tf.reduce_mean(softmaxLoss) # ------ optimizer ------ lrGP_PH,lrC_PH = tf.placeholder(tf.float32,shape=[]),tf.placeholder(tf.float32,shape=[]) optim = util.setOptimizer(opt,loss,lrGP_PH,lrC_PH) # ------ generate summaries ------ summaryImageTrain = [] summaryImageTest = [] if opt.netType=="STN" or opt.netType=="IC-STN": for l in range(opt.warpN+1): summaryImageTrain.append(util.imageSummary(opt,imageWarpAll[l],"TRAIN_warp{0}".format(l),opt.H,opt.W)) summaryImageTest.append(util.imageSummary(opt,imageWarpAll[l],"TEST_warp{0}".format(l),opt.H,opt.W)) summaryImageTrain = tf.summary.merge(summaryImageTrain) summaryImageTest = tf.summary.merge(summaryImageTest) summaryLossTrain = tf.summary.scalar("TRAIN_loss",loss) testErrorPH = tf.placeholder(tf.float32,shape=[]) testImagePH = tf.placeholder(tf.float32,shape=[opt.labelN,opt.H,opt.W,1]) summaryErrorTest = tf.summary.scalar("TEST_error",testErrorPH) if opt.netType=="STN" or opt.netType=="IC-STN": summaryMeanTest0 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_mean_init",opt.H,opt.W) summaryMeanTest1 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_mean_warped",opt.H,opt.W) summaryVarTest0 = util.imageSummaryMeanVar(opt,testImagePH*3,"TEST_var_init",opt.H,opt.W) summaryVarTest1 = util.imageSummaryMeanVar(opt,testImagePH*3,"TEST_var_warped",opt.H,opt.W) # load data print(util.toMagenta("loading MNIST dataset..."))
imageWarpRescale = imageWarp*tf.sqrt(imageVar)+imageMean softmax = tf.nn.softmax(output) labelOnehot = tf.one_hot(label,opt.labelN) prediction = tf.equal(tf.argmax(softmax,1),label) # ------ define loss ------ softmaxLoss = tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labelOnehot) loss = tf.reduce_mean(softmaxLoss) # ------ optimizer ------ lrGP_PH,lrC_PH = tf.placeholder(tf.float32,shape=[]),tf.placeholder(tf.float32,shape=[]) optim = util.setOptimizer(opt,loss,lrGP_PH,lrC_PH) # ------ generate summaries ------ summaryImageTrain = [] summaryImageTest = [] if opt.netType=="STN" or opt.netType=="IC-STN": for l in range(opt.warpN+1): summaryImageTrain.append(util.imageSummary(opt,imageWarpAll[l]*tf.sqrt(imageVar)+imageMean,"TRAIN_warp{0}".format(l),opt.H,opt.W)) summaryImageTest.append(util.imageSummary(opt,imageWarpAll[l]*tf.sqrt(imageVar)+imageMean,"TEST_warp{0}".format(l),opt.H,opt.W)) summaryImageTrain = tf.summary.merge(summaryImageTrain) summaryImageTest = tf.summary.merge(summaryImageTest) summaryLossTrain = tf.summary.scalar("TRAIN_loss",loss) testErrorPH = tf.placeholder(tf.float32,shape=[]) testImagePH = tf.placeholder(tf.float32,shape=[opt.labelN,opt.H,opt.W,3]) summaryErrorTest = tf.summary.scalar("TEST_error",testErrorPH) if opt.netType=="STN" or opt.netType=="IC-STN": summaryMeanTest0 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_mean_init",opt.H,opt.W) summaryMeanTest1 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_mean_warped",opt.H,opt.W) summaryVarTest0 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_var_init",opt.H,opt.W) summaryVarTest1 = util.imageSummaryMeanVar(opt,testImagePH,"TEST_var_warped",opt.H,opt.W) # load data print(util.toMagenta("loading GTSRB dataset..."))