def ICSTN(opt,imageFull,p):
def conv2Layer(opt,feat,outDim):
weight,bias = createVariable(opt,[7,7,int(feat.shape[-1]),outDim],stddev=opt.stdGP)
conv = tf.nn.conv2d(feat,weight,strides=[1,1,1,1],padding="VALID")+bias
return conv
def linearLayer(opt,feat,outDim):
weight,bias = createVariable(opt,[int(feat.shape[-1]),outDim],stddev=opt.stdGP)
fc = tf.matmul(feat,weight)+bias
return fc
imageWarpAll = []
for l in range(opt.warpN):
with tf.variable_scope("geometric",reuse=l>0):
pMtrx = warp.vec2mtrx(opt,p)
imageWarp = warp.transformCropImage(opt,imageFull,pMtrx)
imageWarpAll.append(imageWarp)
feat = imageWarp
with tf.variable_scope("conv1"):
feat = conv2Layer(opt,feat,6)
feat = tf.nn.relu(feat)
with tf.variable_scope("conv2"):
feat = conv2Layer(opt,feat,24)
feat = tf.nn.relu(feat)
feat = tf.reshape(feat,[opt.batchSize,-1])
with tf.variable_scope("fc3"):
feat = linearLayer(opt,feat,opt.warpDim)
dp = feat
p = warp.compose(opt,p,dp)
pMtrx = warp.vec2mtrx(opt,p)
imageWarp = warp.transformCropImage(opt,imageFull,pMtrx)
imageWarpAll.append(imageWarp)
return imageWarpAll
def cSTrecur_depth2_CF(imageInput, p, STlayerN, dimShape, stddev, params):
[STconv1dim] = dimShape
STconv1fcDim = (params.H - 8) * (params.W - 8) * STconv1dim
with tf.name_scope("cSTrecur"):
with tf.variable_scope("conv1"):
weight1, bias1 = createVariable([9, 9, 1, STconv1dim], stddev)
with tf.variable_scope("fc2"):
weight2, bias2 = createVariable([STconv1fcDim, params.pDim], 0,
True)
for l in range(STlayerN):
with tf.name_scope("cSTrecur{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
with tf.variable_scope("conv1"):
STconv1 = tf.nn.conv2d(
ImWarp, weight1, strides=[1, 1, 1, 1
], padding="VALID") + bias1
STrelu1 = tf.nn.relu(STconv1)
STrelu1vec = tf.reshape(STrelu1, [-1, STconv1fcDim])
with tf.variable_scope("fc2"):
STfc2 = tf.matmul(STrelu1vec, weight2) + bias2
p = warp.compose(p, STfc2, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def ST_depth1_F(ImWarp, p, STlayerN, dimShape, stddev, params):
makeImageSummary("image", ImWarp, params)
for l in range(STlayerN):
with tf.name_scope("ST{0}".format(l)):
ImWarpVec = tf.reshape(ImWarp, [-1, params.H * params.W])
with tf.variable_scope("fc1"):
weight, bias = createVariable(
[params.H * params.W, params.pDim], 0, True)
STfc1 = tf.matmul(ImWarpVec, weight) + bias
warpMtrx = warp.vec2mtrxBatch(STfc1, params)
ImWarp = data.ImWarpIm(ImWarp, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
p = warp.compose(p, STfc1, params)
return ImWarp, p
def cSTrecur_depth4_CCFF(imageInput, p, STlayerN, dimShape, stddev, params):
[STconv1dim, STconv2dim, STfc3dim] = dimShape
STconv2fcDim = (params.H - 12) // 2 * (params.W - 12) // 2 * STconv2dim
with tf.name_scope("cSTrecur"):
with tf.variable_scope("conv1"):
weight1, bias1 = createVariable([7, 7, 1, STconv1dim], stddev)
with tf.variable_scope("conv2"):
weight2, bias2 = createVariable([7, 7, STconv1dim, STconv2dim],
stddev)
with tf.variable_scope("fc3"):
weight3, bias3 = createVariable([STconv2fcDim, STfc3dim], stddev)
with tf.variable_scope("fc4"):
weight4, bias4 = createVariable([STfc3dim, params.pDim], 0, True)
ImWarp = imageInput
k = 1
for l in range(STlayerN):
with tf.name_scope("cSTrecur{0}".format(l)):
with tf.variable_scope("conv1"):
STconv1 = tf.nn.conv2d(
ImWarp, weight1, strides=[1, 1, 1, 1
], padding="VALID") + bias1
STrelu1 = tf.nn.relu(STconv1)
with tf.variable_scope("conv2"):
STconv2 = tf.nn.conv2d(
STrelu1, weight2, strides=[1, 1, 1, 1
], padding="VALID") + bias2
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc3"):
STfc3 = tf.matmul(STmaxpool2vec, weight3) + bias3
STrelu3 = tf.nn.relu(STfc3)
with tf.variable_scope("fc4"):
STfc4 = tf.matmul(STrelu3, weight4) + bias4
if k == 1:
p = STfc4
else:
p = warp.compose(p, STfc4, params)
k = k + 1
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def forward(self, opt, image, p):
imageWarpAll = []
for l in range(opt.warpN):
pMtrx = warp.vec2mtrx(opt, p)
imageWarp = warp.transformImage(opt, image, pMtrx)
imageWarpAll.append(imageWarp)
feat = imageWarp
feat = self.conv2Layers(feat).view(opt.batchSize, -1)
feat = self.linearLayers(feat)
dp = feat
p = warp.compose(opt, p, dp)
pMtrx = warp.vec2mtrx(opt, p)
imageWarp = warp.transformImage(opt, image, pMtrx)
imageWarpAll.append(imageWarp)
return imageWarpAll
def PT_STN(image, p, stdGP, warpN, batchSize, dataH, dataW):
shape = image.shape
appendarr = tf.ones([shape[0], shape[1], shape[2], 1])
image = tf.concat((image, appendarr), axis=3)
with tf.variable_scope("pt"):
dp = None
# define recurrent spatial transformations
for l in range(warpN):
with tf.variable_scope("warp{0}".format(l)):
pMtrx = warp.vec2mtrx(batchSize, p)
imagewarp = warp.transformImage(batchSize, image, pMtrx, dataH,
dataW)
# geometric predictor
imageConcat = imagewarp
feat = imageConcat
with tf.variable_scope("conv1"):
feat, imageConcat = conv2Layer(feat, imageConcat, 32,
stdGP) # 72x72
with tf.variable_scope("conv2"):
feat, imageConcat = conv2Layer(feat, imageConcat, 64,
stdGP) # 36x36
with tf.variable_scope("conv3"):
feat, imageConcat = conv2Layer(feat, imageConcat, 128,
stdGP) # 18x18
with tf.variable_scope("conv4"):
feat, imageConcat = conv2Layer(feat, imageConcat, 256,
stdGP) # 9x9
with tf.variable_scope("conv5"):
feat, imageConcat = conv2Layer(feat, imageConcat, 512,
stdGP) # 5x5
feat = tf.reshape(feat, [batchSize, -1])
with tf.variable_scope("fc6"):
feat = linearLayer(feat, 256, stdGP)
with tf.variable_scope("fc7"):
feat = linearLayer(feat, 8, stdGP, final=True)
dp = feat
p = warp.compose(p, dp)
# warp image with final p
#print(imagewarp.shape)
pMtrx = warp.vec2mtrx(batchSize, p)
imagewarp = warp.transformImage(batchSize, imagewarp, pMtrx, dataH,
dataW)
colorFG, maskFG = imagewarp[:, :, :, :3], imagewarp[:, :, :, 3:]
imagewarp = colorFG * maskFG
#print(imagewarp.shape)
return imagewarp
def geometric_multires(opt,imageBG,imageFG,p):
def downsample(x):
padH,padW = int(x.shape[1])%2,int(x.shape[2])%2
if padH!=0 or padW!=0: x = tf.pad(x,[[0,0],[0,padH],[0,padW],[0,0]])
return tf.nn.avg_pool(x,[1,2,2,1],[1,2,2,1],"VALID")
def conv2Layer(opt,feat,imageConcat,outDim,final=False):
weight,bias = createVariable(opt,[4,4,int(feat.shape[-1]),outDim],stddev=opt.stdGP)
conv = tf.nn.conv2d(feat,weight,strides=[1,2,2,1],padding="SAME")+bias
feat = tf.nn.relu(conv)
imageConcat = downsample(imageConcat)
feat = tf.concat([feat,imageConcat],axis=3)
return (feat if not final else conv),imageConcat
def linearLayer(opt,feat,outDim,final=False):
weight,bias = createVariable(opt,[int(feat.shape[-1]),outDim],stddev=opt.stdGP)
fc = tf.matmul(feat,weight)+bias
feat = tf.nn.relu(fc)
return feat if not final else fc
with tf.variable_scope("geometric"):
imageFGwarpAll,pAll = [],[p]
dp = None
# define recurrent spatial transformations
for l in range(opt.warpN):
with tf.variable_scope("warp{0}".format(l)):
pMtrx = warp.vec2mtrx(opt,p)
imageFGwarp = warp.transformImage(opt,imageFG,pMtrx)
imageFGwarpAll.append(imageFGwarp)
# geometric predictor
imageConcat = tf.concat([imageBG,imageFGwarp],axis=3)
feat = imageConcat # 144x144
with tf.variable_scope("conv1"): feat,imageConcat = conv2Layer(opt,feat,imageConcat,32) # 72x72
with tf.variable_scope("conv2"): feat,imageConcat = conv2Layer(opt,feat,imageConcat,64) # 36x36
with tf.variable_scope("conv3"): feat,imageConcat = conv2Layer(opt,feat,imageConcat,128) # 18x18
with tf.variable_scope("conv4"): feat,imageConcat = conv2Layer(opt,feat,imageConcat,256) # 9x9
with tf.variable_scope("conv5"): feat,imageConcat = conv2Layer(opt,feat,imageConcat,512) # 5x5
feat = tf.reshape(feat,[opt.batchSize,-1])
with tf.variable_scope("fc6"): feat = linearLayer(opt,feat,256)
with tf.variable_scope("fc7"): feat = linearLayer(opt,feat,opt.warpDim,final=True)
dp = feat
p = warp.compose(opt,p,dp)
pAll.append(p)
# warp image with final p
pMtrx = warp.vec2mtrx(opt,p)
imageFGwarp = warp.transformImage(opt,imageFG,pMtrx)
imageFGwarpAll.append(imageFGwarp)
return imageFGwarpAll,pAll,dp
def cSTrecur_depth1_F(imageInput, p, STlayerN, dimShape, stddev, params):
with tf.name_scope("cSTrecur"):
with tf.variable_scope("fc1"):
weight, bias = createVariable([params.H * params.W, params.pDim],
0, True)
for l in range(STlayerN):
with tf.name_scope("cSTrecur{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
imageWarpVec = tf.reshape(ImWarp, [-1, params.H * params.W])
with tf.variable_scope("fc1"):
STfc1 = tf.matmul(imageWarpVec, weight) + bias
p = warp.compose(p, STfc1, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def cSTN(opt, imageInput, p, STlayerN, dimShape, stddev):
for l in range(STlayerN):
with tf.name_scope("cST{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, opt)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, opt)
util.makeImageSummary("imageST{0}".format(l), ImWarp, opt)
[STconv1dim, STconv2dim, STfc3dim] = dimShape
STconv2fcDim = (opt.H - 12) // 2 * (opt.W - 12) // 2 * STconv2dim
with tf.variable_scope("conv1"):
weight, bias = createVariable([7, 7, 1, STconv1dim], stddev)
STconv1 = tf.nn.conv2d(
ImWarp, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu1 = tf.nn.relu(STconv1)
with tf.variable_scope("conv2"):
weight, bias = createVariable([7, 7, STconv1dim, STconv2dim],
stddev)
STconv2 = tf.nn.conv2d(
STrelu1, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc3"):
weight, bias = createVariable([STconv2fcDim, STfc3dim], stddev)
STfc3 = tf.matmul(STmaxpool2vec, weight) + bias
STrelu3 = tf.nn.relu(STfc3)
with tf.variable_scope("fc4"):
weight, bias = createVariable([STfc3dim, opt.pDim], 0, True)
STfc4 = tf.matmul(STrelu3, weight) + bias
p = warp.compose(p, STfc4, opt)
warpMtrx = warp.vec2mtrxBatch(p, opt)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, opt)
util.makeImageSummary("imageST{0}".format(STlayerN), ImWarp, opt)
return ImWarp, p
def ST_depth4_CCFF(ImWarp, p, STlayerN, dimShape, stddev, params):
makeImageSummary("image", ImWarp, params)
for l in range(STlayerN):
with tf.name_scope("ST{0}".format(l)):
[STconv1dim, STconv2dim, STfc3dim] = dimShape
STconv2fcDim = (params.H - 12) // 2 * (params.W -
12) // 2 * STconv2dim
with tf.variable_scope("conv1"):
weight, bias = createVariable([7, 7, 1, STconv1dim], stddev)
STconv1 = tf.nn.conv2d(
ImWarp, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu1 = tf.nn.relu(STconv1)
with tf.variable_scope("conv2"):
weight, bias = createVariable([7, 7, STconv1dim, STconv2dim],
stddev)
STconv2 = tf.nn.conv2d(
STrelu1, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc3"):
weight, bias = createVariable([STconv2fcDim, STfc3dim], stddev)
STfc3 = tf.matmul(STmaxpool2vec, weight) + bias
STrelu3 = tf.nn.relu(STfc3)
with tf.variable_scope("fc4"):
weight, bias = createVariable([STfc3dim, params.pDim], 0, True)
STfc4 = tf.matmul(STrelu3, weight) + bias
warpMtrx = warp.vec2mtrxBatch(STfc4, params)
ImWarp = data.ImWarpIm(ImWarp, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
p = warp.compose(p, STfc4, params)
return ImWarp, p
def geometric_multires(opt, imageBG, imageFG, p):
"""
build geometric predictor
기하학적 예측자(생성자) 설계하는 함수
:param opt: 옵션들
:param imageBG: 백그라운드(인물) 이미지 데이터셋
:param imageFG: 포그라운드(안경) 이미지 데이터셋
:param p: warp 파라미터
:return imageFGwarpAll:
pAll:
dp:
"""
def downsample(x):
"""
차원을 줄이는 함수(레이어)
:param x: 차원을 줄일 대상, input
:return: 평균 풀링을 수행한 중간 output matrix
"""
# 패딩할 높이와 너비를 구함
# 2의 나머지로 구하는 이유는 패딩 크기가 2*2이고 stride가 2이기 때문에
# 높이 너비가 짝수여야 원할함 => 짝수로 맞춰주기 위해 패딩 수행
padH, padW = int(x.shape[1]) % 2, int(x.shape[2]) % 2
# 패딩할 높이와 너비가 0이 아니라면 패딩 수행
if padH != 0 or padW != 0:
x = tf.pad(x, [[0, 0], [0, padH], [0, padW], [0, 0]])
# 패딩한 데이터를 평균 풀링하여 출력
return tf.nn.avg_pool(x, [1, 2, 2, 1], [1, 2, 2, 1], "VALID")
# convolutional 함수(레이어)
def conv2Layer(opt, feat, imageConcat, outDim):
weight, bias = createVariable(opt,
[4, 4, int(feat.shape[-1]), outDim],
stddev=opt.stdGP)
conv = tf.nn.conv2d(feat, weight, strides=[1, 2, 2, 1],
padding="SAME") + bias
feat = tf.nn.relu(conv)
imageConcat = downsample(imageConcat)
feat = tf.concat([feat, imageConcat], axis=3)
return feat, imageConcat
def linearLayer(opt, feat, outDim, final=False):
weight, bias = createVariable(opt, [int(feat.shape[-1]), outDim],
stddev=opt.stdGP)
fc = tf.matmul(feat, weight) + bias
feat = tf.nn.relu(fc)
return feat if not final else fc
with tf.variable_scope("geometric"):
imageFGwarpAll, pAll = [], [p]
dp = None
# define spatial transformations
for l in range(opt.warpN):
with tf.variable_scope("warp{0}".format(l)):
pMtrx = warp.vec2mtrx(opt, p)
imageFGwarp = warp.transformImage(opt, imageFG, pMtrx)
imageFGwarpAll.append(imageFGwarp)
# geometric predictor
imageConcat = tf.concat([imageBG, imageFGwarp], axis=3)
feat = imageConcat # 144x144
with tf.variable_scope("conv1"):
feat, imageConcat = conv2Layer(opt, feat, imageConcat,
32) # 72x72
with tf.variable_scope("conv2"):
feat, imageConcat = conv2Layer(opt, feat, imageConcat,
64) # 36x36
with tf.variable_scope("conv3"):
feat, imageConcat = conv2Layer(opt, feat, imageConcat,
128) # 18x18
with tf.variable_scope("conv4"):
feat, imageConcat = conv2Layer(opt, feat, imageConcat,
256) # 9x9
with tf.variable_scope("conv5"):
feat, imageConcat = conv2Layer(opt, feat, imageConcat,
512) # 5x5
feat = tf.reshape(feat, [opt.batchSize, -1])
with tf.variable_scope("fc6"):
feat = linearLayer(opt, feat, 256)
with tf.variable_scope("fc7"):
feat = linearLayer(opt, feat, opt.warpDim, final=True)
dp = feat
p = warp.compose(opt, p, dp)
pAll.append(p)
# warp image with final p
pMtrx = warp.vec2mtrx(opt, p)
imageFGwarp = warp.transformImage(opt, imageFG, pMtrx)
imageFGwarpAll.append(imageFGwarp)
return imageFGwarpAll, pAll, dp
