def cSTrecur_depth2_CF(imageInput, p, STlayerN, dimShape, stddev, params):
[STconv1dim] = dimShape
STconv1fcDim = (params.H - 8) * (params.W - 8) * STconv1dim
with tf.name_scope("cSTrecur"):
with tf.variable_scope("conv1"):
weight1, bias1 = createVariable([9, 9, 1, STconv1dim], stddev)
with tf.variable_scope("fc2"):
weight2, bias2 = createVariable([STconv1fcDim, params.pDim], 0,
True)
for l in range(STlayerN):
with tf.name_scope("cSTrecur{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
with tf.variable_scope("conv1"):
STconv1 = tf.nn.conv2d(
ImWarp, weight1, strides=[1, 1, 1, 1
], padding="VALID") + bias1
STrelu1 = tf.nn.relu(STconv1)
STrelu1vec = tf.reshape(STrelu1, [-1, STconv1fcDim])
with tf.variable_scope("fc2"):
STfc2 = tf.matmul(STrelu1vec, weight2) + bias2
p = warp.compose(p, STfc2, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def cSTrecur_depth4_CCFF(imageInput, p, STlayerN, dimShape, stddev, params):
[STconv1dim, STconv2dim, STfc3dim] = dimShape
STconv2fcDim = (params.H - 12) // 2 * (params.W - 12) // 2 * STconv2dim
with tf.name_scope("cSTrecur"):
with tf.variable_scope("conv1"):
weight1, bias1 = createVariable([7, 7, 1, STconv1dim], stddev)
with tf.variable_scope("conv2"):
weight2, bias2 = createVariable([7, 7, STconv1dim, STconv2dim],
stddev)
with tf.variable_scope("fc3"):
weight3, bias3 = createVariable([STconv2fcDim, STfc3dim], stddev)
with tf.variable_scope("fc4"):
weight4, bias4 = createVariable([STfc3dim, params.pDim], 0, True)
ImWarp = imageInput
k = 1
for l in range(STlayerN):
with tf.name_scope("cSTrecur{0}".format(l)):
with tf.variable_scope("conv1"):
STconv1 = tf.nn.conv2d(
ImWarp, weight1, strides=[1, 1, 1, 1
], padding="VALID") + bias1
STrelu1 = tf.nn.relu(STconv1)
with tf.variable_scope("conv2"):
STconv2 = tf.nn.conv2d(
STrelu1, weight2, strides=[1, 1, 1, 1
], padding="VALID") + bias2
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc3"):
STfc3 = tf.matmul(STmaxpool2vec, weight3) + bias3
STrelu3 = tf.nn.relu(STfc3)
with tf.variable_scope("fc4"):
STfc4 = tf.matmul(STrelu3, weight4) + bias4
if k == 1:
p = STfc4
else:
p = warp.compose(p, STfc4, params)
k = k + 1
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def genPerturbations(params):
with tf.name_scope("genPerturbations"):
X = np.tile(params.canon4pts[:, 0], [params.batchSize, 1])
Y = np.tile(params.canon4pts[:, 1], [params.batchSize, 1])
dX = tf.random_normal([params.batchSize, 4]) * params.warpScale["pert"] \
+ tf.random_normal([params.batchSize, 1]) * params.warpScale["trans"]
dY = tf.random_normal([params.batchSize, 4]) * params.warpScale["pert"] \
+ tf.random_normal([params.batchSize, 1]) * params.warpScale["trans"]
O = np.zeros([params.batchSize, 4], dtype=np.float32)
I = np.ones([params.batchSize, 4], dtype=np.float32)
# fit warp parameters to generated displacements
if params.warpType == "affine":
J = np.concatenate([np.stack([X, Y, I, O, O, O], axis=-1),
np.stack([O, O, O, X, Y, I], axis=-1)], axis=1)
dXY = tf.expand_dims(tf.concat([dX, dY], 1), -1)
dpBatch = tf.matrix_solve_ls(J, dXY)
elif params.warpType == "homography":
A = tf.concat([tf.stack([X, Y, I, O, O, O, -X * (X + dX), -Y * (X + dX)], axis=-1),
tf.stack([O, O, O, X, Y, I, -X * (Y + dY), -Y * (Y + dY)], axis=-1)], 1)
b = tf.expand_dims(tf.concat([X + dX, Y + dY], 1), -1)
dpBatch = tf.matrix_solve_ls(A, b)
dpBatch -= tf.to_float(tf.reshape([1, 0, 0, 0, 1, 0, 0, 0], [1, 8, 1]))
dpBatch = tf.reduce_sum(dpBatch, reduction_indices=-1)
dpMtrxBatch = warp.vec2mtrxBatch(dpBatch, params)
return dpBatch
def cST_depth1_F(imageInput, p, STlayerN, dimShape, stddev, params):
for l in range(STlayerN):
with tf.name_scope("cST{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
imageWarpVec = tf.reshape(ImWarp, [-1, params.H * params.W])
with tf.variable_scope("fc1"):
weight, bias = createVariable(
[params.H * params.W, params.pDim], 0, True)
STfc1 = tf.matmul(imageWarpVec, weight) + bias
p = warp.compose(p, STfc1, params)
warpMtrx = warp.vec2mtrxBatch(p, params)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, params)
makeImageSummary("imageST{0}".format(STlayerN), ImWarp, params)
return ImWarp, p
def MYST_depth4_CCFF(ImWarp, STlayerN, dimShape, stddev, params):
makeImageSummary("image", ImWarp, params)
for l in range(STlayerN):
with tf.name_scope("ST{0}".format(l)):
[STconv1dim, STconv2dim] = dimShape
STconv2fcDim = 6 * 6 * STconv2dim
with tf.variable_scope("conv1"):
weight, bias = createVariable([3, 3, 1, STconv1dim], stddev)
STconv1 = tf.nn.conv2d(
ImWarp, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu1 = tf.nn.relu(STconv1)
STmaxpool1 = tf.nn.max_pool(STrelu1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
with tf.variable_scope("conv2"):
weight, bias = createVariable([4, 4, STconv1dim, STconv2dim],
stddev)
STconv2 = tf.nn.conv2d(
STmaxpool1, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc4"):
weight, bias = createVariable([STconv2fcDim, params.pDim], 0,
True)
STfc4 = tf.matmul(STmaxpool2vec, weight) + bias
warpMtrx = warp.vec2mtrxBatch(STfc4, params)
ImWarp = data.ImWarpIm(ImWarp, warpMtrx, params)
makeImageSummary("imageST{0}".format(l), ImWarp, params)
def genPerturbations(opt):
with tf.name_scope("genPerturbations"):
X = np.tile(opt.canon4pts[:,0],[opt.batchSize,1])
Y = np.tile(opt.canon4pts[:,1],[opt.batchSize,1])
dX = tf.random_normal([opt.batchSize,4])*opt.warpScale["pert"] \
+tf.random_normal([opt.batchSize,1])*opt.warpScale["trans"]
dY = tf.random_normal([opt.batchSize,4])*opt.warpScale["pert"] \
+tf.random_normal([opt.batchSize,1])*opt.warpScale["trans"]
O = np.zeros([opt.batchSize,4],dtype=np.float32)
I = np.ones([opt.batchSize,4],dtype=np.float32)
# fit warp parameters to generated displacements
if opt.warpType == "affine":
J = np.concatenate([np.stack([X,Y,I,O,O,O],axis=-1),
np.stack([O,O,O,X,Y,I],axis=-1)],axis=1)
dXY = tf.expand_dims(tf.concat(1,[dX,dY]),-1)
dpBatch = tf.matrix_solve_ls(J,dXY)
elif opt.warpType == "homography":
A = tf.concat([tf.stack([X,Y,I,O,O,O,-X*(X+dX),-Y*(X+dX)],axis=-1),
tf.stack([O,O,O,X,Y,I,-X*(Y+dY),-Y*(Y+dY)],axis=-1)],1)
b = tf.expand_dims(tf.concat([X+dX,Y+dY],1),-1)
dpBatch = tf.matrix_solve_ls(A,b)
dpBatch -= tf.to_float(tf.reshape([1,0,0,0,1,0,0,0],[1,8,1]))
dpBatch = tf.reduce_sum(dpBatch,reduction_indices=-1)
dpMtrxBatch = warp.vec2mtrxBatch(dpBatch,opt)
return dpBatch
def cSTN(opt, imageInput, p, STlayerN, dimShape, stddev):
for l in range(STlayerN):
with tf.name_scope("cST{0}".format(l)):
warpMtrx = warp.vec2mtrxBatch(p, opt)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, opt)
util.makeImageSummary("imageST{0}".format(l), ImWarp, opt)
[STconv1dim, STconv2dim, STfc3dim] = dimShape
STconv2fcDim = (opt.H - 12) // 2 * (opt.W - 12) // 2 * STconv2dim
with tf.variable_scope("conv1"):
weight, bias = createVariable([7, 7, 1, STconv1dim], stddev)
STconv1 = tf.nn.conv2d(
ImWarp, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu1 = tf.nn.relu(STconv1)
with tf.variable_scope("conv2"):
weight, bias = createVariable([7, 7, STconv1dim, STconv2dim],
stddev)
STconv2 = tf.nn.conv2d(
STrelu1, weight, strides=[1, 1, 1, 1
], padding="VALID") + bias
STrelu2 = tf.nn.relu(STconv2)
STmaxpool2 = tf.nn.max_pool(STrelu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID")
STmaxpool2vec = tf.reshape(STmaxpool2, [-1, STconv2fcDim])
with tf.variable_scope("fc3"):
weight, bias = createVariable([STconv2fcDim, STfc3dim], stddev)
STfc3 = tf.matmul(STmaxpool2vec, weight) + bias
STrelu3 = tf.nn.relu(STfc3)
with tf.variable_scope("fc4"):
weight, bias = createVariable([STfc3dim, opt.pDim], 0, True)
STfc4 = tf.matmul(STrelu3, weight) + bias
p = warp.compose(p, STfc4, opt)
warpMtrx = warp.vec2mtrxBatch(p, opt)
ImWarp = data.imageWarpIm(imageInput, warpMtrx, opt)
util.makeImageSummary("imageST{0}".format(STlayerN), ImWarp, opt)
return ImWarp, p
print("warpType: {0}".format(params.warpType))
print("batchSize: {0}".format(params.batchSize))
print("GPU device: {0}".format(args.gpu))
print("------------------------------------------")
tf.reset_default_graph()
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
# build graph
with tf.device(params.GPUdevice):
# generate training data on the fly
imageRawBatch = tf.placeholder(tf.float32,
shape=[None, 28, 28],
name="image")
pInitBatch = data.genPerturbations(params)
pInitMtrxBatch = warp.vec2mtrxBatch(pInitBatch, params)
ImBatch = data.imageWarpIm(imageRawBatch,
pInitMtrxBatch,
params,
name=None)
# build network
if args.type == "CNN":
outputBatch = graph.buildFullCNN(ImBatch, [3, 6, 9, 12, 48], 0.1,
params)
elif args.type == "STN":
ImWarpBatch, pBatch = graphST.ST_depth4_CCFF(ImBatch, pInitBatch, 1,
[4, 8, 48], 0.01, params)
outputBatch = graph.buildCNN(ImWarpBatch, [3], 0.03, params)
elif args.type == "cSTN":
ImWarpBatch, pBatch = graphST.cST_depth4_CCFF(imageRawBatch,
pInitBatch, 1,
print("warpType: {0}".format(params.warpType))
print("batchSize: {0}".format(params.batchSize))
print("GPU device: {0}".format(args.gpu))
print("------------------------------------------")
tf.reset_default_graph()
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
# build graph
with tf.device(params.GPUdevice):
# generate training data on the fly
imageRawBatch = tf.placeholder(tf.float32,
shape=[None, 32, 32, None],
name="image")
pInitBatch = data.genPerturbations(params) #produce raodong
pInitMtrxBatch = warp.vec2mtrxBatch(
pInitBatch, params) #jiang rao dong can shu zhuan hua wei ju zhen
ImBatch = data.imageWarpIm(
imageRawBatch, pInitMtrxBatch, params,
name=None) #cong mei pi tuxiang zhuanhua wei bianhua hou de tuxiang
# build network
if args.type == "CNN":
outputBatch = graph.buildFullCNN(imageRawBatch, [64, 64, 128, 300],
0.1, params)
elif args.type == "STN":
ImWarpBatch, pBatch = graphST.ST_depth1_F(imageRawBatch, pInitBatch, 1,
[65], 0.01, params)
outputBatch = graph.buildFullCNN(ImWarpBatch, [64, 64, 128, 300], 0.03,
params)
elif args.type == "cSTN":
ImWarpBatch, pBatch = graphST.cST_depth4_CCFF(imageRawBatch,
pInitBatch, 1,
print("------------------------------------------")
print("warpScale: (pert) {0} (trans) {1}".format(opt.warpScale["pert"],opt.warpScale["trans"]))
print("warpType: {0}".format(opt.warpType))
print("batchSize: {0}".format(opt.batchSize))
print("GPU device: {0}".format(opt.gpu))
print("------------------------------------------")
tf.reset_default_graph()
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
# build graph
with tf.device(opt.GPUdevice):
# generate training data on the fly
imageRawBatch = tf.placeholder(tf.float32,shape=[None,28,28],name="image")
pInitBatch = data.genPerturbations(opt)
pInitMtrxBatch = warp.vec2mtrxBatch(pInitBatch,opt)
ImBatch = data.imageWarpIm(imageRawBatch,pInitMtrxBatch,opt,name=None)
# build network
if opt.type=="CNN":
outputBatch = graph.fullCNN(opt,ImBatch,[3,6,9,12,48],0.1)
elif opt.type=="STN":
ImWarpBatch,pBatch = graph.STN(opt,ImBatch,pInitBatch,1,[4,8,48],0.01)
outputBatch = graph.CNN(opt,ImWarpBatch,[3],0.03)
elif opt.type=="cSTN":
ImWarpBatch,pBatch = graph.cSTN(opt,imageRawBatch,pInitBatch,1,[4,8,48],0.01)
outputBatch = graph.CNN(opt,ImWarpBatch,[3],0.03)
elif opt.type=="ICSTN":
ImWarpBatch,pBatch = graph.ICSTN(opt,imageRawBatch,pInitBatch,opt.recurN,[4,8,48],0.01)
outputBatch = graph.CNN(opt,ImWarpBatch,[3],0.03)
# define loss/optimizer/summaries
imageSummaries = tf.summary.merge_all()
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
X = np.tile(params.canon4pts[:, 0], [params.batchSize, 1])
Y = np.tile(params.canon4pts[:, 1], [params.batchSize, 1])
dX = tf.random_normal([params.batchSize, 4]) * params.warpScale["pert"] \
+ tf.random_normal([params.batchSize, 1]) * params.warpScale["trans"]
dY = tf.random_normal([params.batchSize, 4]) * params.warpScale["pert"] \
+ tf.random_normal([params.batchSize, 1]) * params.warpScale["trans"]
O = np.zeros([params.batchSize, 4], dtype=np.float32)
I = np.ones([params.batchSize, 4], dtype=np.float32)
# fit warp parameters to generated displacements
if params.warpType == "affine":
J = np.concatenate([
np.stack([X, Y, I, O, O, O], axis=-1),
np.stack([O, O, O, X, Y, I], axis=-1)
],
axis=1)
dXY = tf.expand_dims(tf.concat([dX, dY], 1), -1)
dpBatch = tf.matrix_solve_ls(J, dXY)
elif params.warpType == "homography":
A = tf.concat([
tf.stack([X, Y, I, O, O, O, -X * (X + dX), -Y * (X + dX)], axis=-1),
tf.stack([O, O, O, X, Y, I, -X * (Y + dY), -Y * (Y + dY)], axis=-1)
], 1)
b = tf.expand_dims(tf.concat([X + dX, Y + dY], 1), -1)
dpBatch = tf.matrix_solve_ls(A, b)
dpBatch -= tf.to_float(tf.reshape([1, 0, 0, 0, 1, 0, 0, 0], [1, 8, 1]))
dpBatch = tf.reduce_sum(dpBatch, reduction_indices=-1)
dpMtrxBatch = warp.vec2mtrxBatch(dpBatch, params)
