def model():
y1_dim = 50
y2_dim = 100
learning_rate = 0.01
y1 = tf.placeholder('float32', [None, y1_dim])
y2 = tf.placeholder('float32', [None, y2_dim])
start1 = StartNode(input_vars=[y1])
start2 = StartNode(input_vars=[y2])
h1 = HiddenNode(prev=[start1, start2],
input_merge_mode=Concat(),
layers=[Linear(y1_dim + y2_dim, y2_dim),
RELU()])
h2 = HiddenNode(prev=[start2], layers=[Linear(y2_dim, y2_dim), RELU()])
h3 = HiddenNode(prev=[h1, h2],
input_merge_mode=Sum(),
layers=[Linear(y2_dim, y1_dim),
RELU()])
e1 = EndNode(prev=[h3])
e2 = EndNode(prev=[h2])
graph = Graph(start=[start1, start2], end=[e1, e2])
o1, o2 = graph.train_fprop()
o1_mse = tf.reduce_mean((y1 - o1)**2)
o2_mse = tf.reduce_mean((y2 - o2)**2)
mse = o1_mse + o2_mse
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(mse)
return y1, y2, o1, o2, optimizer
def model():
with tf.name_scope('MnistCNN'):
seq = tg.Sequential()
seq.add(Conv2D(num_filters=32, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(BatchNormalization())
seq.add(RELU())
seq.add(MaxPooling(poolsize=(2, 2), stride=(2,2), padding='SAME'))
seq.add(LRN())
seq.add(Conv2D(num_filters=64, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(BatchNormalization())
seq.add(RELU())
seq.add(MaxPooling(poolsize=(2, 2), stride=(2,2), padding='SAME'))
seq.add(LRN())
seq.add(Flatten())
seq.add(Linear(128))
seq.add(BatchNormalization())
seq.add(Tanh())
seq.add(Dropout(0.8))
seq.add(Linear(256))
seq.add(BatchNormalization())
seq.add(Tanh())
seq.add(Dropout(0.8))
seq.add(Linear(10))
seq.add(Softmax())
return seq
def model3D(img=(84, 256, 256)):
with tf.name_scope('WMH'):
seq = tg.Sequential()
convStride = (1, 1, 1)
poolStride = (2, 2, 2)
seq.add(
Conv3D(input_channels=1,
num_filters=10,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
seq.add(TFBatchNormalization(name='b1'))
seq.add(
MaxPool3D(poolsize=(2, 2, 2), stride=poolStride, padding='SAME'))
layerSize1 = updateConvLayerSize(img, poolStride)
#print("layer1: "+str(layerSize1))
seq.add(RELU())
seq.add(
Conv3D(input_channels=10,
num_filters=20,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
seq.add(TFBatchNormalization(name='b2'))
#layerSize2 = updateConvLayerSize(layerSize1,convStride)
#print("layer1: "+str(layerSize2))
seq.add(RELU())
seq.add(
Conv3D_Tranpose1(input_channels=20,
num_filters=10,
output_shape=layerSize1,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
seq.add(RELU())
seq.add(
Conv3D_Tranpose1(input_channels=10,
num_filters=3,
output_shape=img,
kernel_size=(3, 3, 3),
stride=(2, 2, 2),
padding='SAME'))
##
seq.add(RELU())
seq.add(
Conv3D(input_channels=3,
num_filters=2,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
##
seq.add(Softmax())
#seq.add(Sigmoid())
return seq
def model(word_len, sent_len, nclass):
unicode_size = 1000
ch_embed_dim = 20
h, w = valid(ch_embed_dim,
word_len,
stride=(1, 1),
kernel_size=(ch_embed_dim, 5))
h, w = valid(h, w, stride=(1, 1), kernel_size=(1, 5))
h, w = valid(h, w, stride=(1, 2), kernel_size=(1, 5))
conv_out_dim = int(h * w * 60)
X_ph = tf.placeholder('int32', [None, sent_len, word_len])
input_sn = tg.StartNode(input_vars=[X_ph])
charcnn_hn = tg.HiddenNode(prev=[input_sn],
layers=[
Reshape(shape=(-1, word_len)),
Embedding(cat_dim=unicode_size,
encode_dim=ch_embed_dim,
zero_pad=True),
Reshape(shape=(-1, ch_embed_dim, word_len,
1)),
Conv2D(input_channels=1,
num_filters=20,
padding='VALID',
kernel_size=(ch_embed_dim, 5),
stride=(1, 1)),
RELU(),
Conv2D(input_channels=20,
num_filters=40,
padding='VALID',
kernel_size=(1, 5),
stride=(1, 1)),
RELU(),
Conv2D(input_channels=40,
num_filters=60,
padding='VALID',
kernel_size=(1, 5),
stride=(1, 2)),
RELU(),
Flatten(),
Linear(conv_out_dim, nclass),
Reshape((-1, sent_len, nclass)),
ReduceSum(1),
Softmax()
])
output_en = tg.EndNode(prev=[charcnn_hn])
graph = tg.Graph(start=[input_sn], end=[output_en])
y_train_sb = graph.train_fprop()[0]
y_test_sb = graph.test_fprop()[0]
return X_ph, y_train_sb, y_test_sb
def __init__(self, nclass, h, w, c):
layers = []
identityblk = IdentityBlock(input_channels=c,
input_shape=[h, w],
nlayers=10)
layers.append(identityblk)
layers.append(
Conv2D(input_channels=c,
num_filters=16,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(BatchNormalization(input_shape=[h, w, 16]))
denseblk = DenseBlock(input_channels=16,
input_shape=[h, w],
growth_rate=4,
nlayers=4)
layers.append(denseblk)
layers.append(
Conv2D(input_channels=denseblk.output_channels,
num_filters=32,
kernel_size=(3, 3),
stride=(2, 2),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(2, 2), kernel_size=(3, 3))
layers.append(Dropout(0.5))
layers.append(
Conv2D(input_channels=32,
num_filters=nclass,
kernel_size=(1, 1),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(1, 1))
layers.append(BatchNormalization(input_shape=[h, w, nclass]))
layers.append(
AvgPooling(poolsize=(h, w), stride=(1, 1), padding='VALID'))
layers.append(Flatten())
layers.append(Softmax())
self.startnode = tg.StartNode(input_vars=[None])
model_hn = tg.HiddenNode(prev=[self.startnode], layers=layers)
self.endnode = tg.EndNode(prev=[model_hn])
def model(nclass, h, w, c):
with tf.name_scope('Cifar10AllCNN'):
seq = tg.Sequential()
seq.add(Conv2D(input_channels=c, num_filters=96, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(RELU())
seq.add(TFBatchNormalization(name='b1'))
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(3,3))
seq.add(Conv2D(input_channels=96, num_filters=96, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(RELU())
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(3,3))
seq.add(Dropout(0.5))
seq.add(Conv2D(input_channels=96, num_filters=96, kernel_size=(3, 3), stride=(2, 2), padding='SAME'))
seq.add(RELU())
seq.add(TFBatchNormalization(name='b3'))
h, w = same(in_height=h, in_width=w, stride=(2,2), kernel_size=(3,3))
seq.add(Conv2D(input_channels=96, num_filters=192, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(RELU())
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(3,3))
seq.add(Dropout(0.5))
seq.add(Conv2D(input_channels=192, num_filters=192, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(RELU())
seq.add(TFBatchNormalization(name='b5'))
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(3,3))
seq.add(Conv2D(input_channels=192, num_filters=192, kernel_size=(3, 3), stride=(2, 2), padding='SAME'))
seq.add(RELU())
h, w = same(in_height=h, in_width=w, stride=(2,2), kernel_size=(3,3))
seq.add(Dropout(0.5))
seq.add(Conv2D(input_channels=192, num_filters=192, kernel_size=(3, 3), stride=(1, 1), padding='SAME'))
seq.add(RELU())
seq.add(TFBatchNormalization(name='b7'))
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(3,3))
seq.add(Conv2D(input_channels=192, num_filters=192, kernel_size=(1, 1), stride=(1, 1), padding='SAME'))
seq.add(RELU())
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(1,1))
seq.add(Dropout(0.5))
seq.add(Conv2D(input_channels=192, num_filters=nclass, kernel_size=(1, 1), stride=(1, 1), padding='SAME'))
seq.add(RELU())
seq.add(TFBatchNormalization(name='b9'))
h, w = same(in_height=h, in_width=w, stride=(1,1), kernel_size=(1,1))
seq.add(AvgPooling(poolsize=(h, w), stride=(1,1), padding='VALID'))
seq.add(Flatten())
seq.add(Softmax())
return seq
def __init__(self, h, w, c):
layers1 = []
layers1.append(Conv2D(input_channels=c, num_filters=1, kernel_size=(2,2), stride=(1,1), padding='SAME'))
layers1.append(BatchNormalization(input_shape=[h,w,1]))
layers1.append(RELU())
layers2 = []
layers2.append(Conv2D(input_channels=c, num_filters=1, kernel_size=(2,2), stride=(1,1), padding='SAME'))
layers2.append(BatchNormalization(input_shape=[h,w,1]))
layers2.append(RELU())
self.startnode = tg.StartNode(input_vars=[None])
hn1 = tg.HiddenNode(prev=[self.startnode], layers=layers1)
hn2 = tg.HiddenNode(prev=[self.startnode], layers=layers2)
hn3 = tg.HiddenNode(prev=[hn1, hn2], input_merge_mode=Sum())
self.endnode = tg.EndNode(prev=[hn3])
def model():
with tf.name_scope('MnistCNN'):
seq = tg.Sequential()
seq.add(
Conv2D(input_channels=1,
num_filters=32,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
h, w = same(in_height=28,
in_width=28,
stride=(1, 1),
kernel_size=(3, 3))
seq.add(BatchNormalization(input_shape=[h, w, 32]))
seq.add(RELU())
seq.add(MaxPooling(poolsize=(2, 2), stride=(2, 2), padding='SAME'))
h, w = same(in_height=h, in_width=w, stride=(2, 2), kernel_size=(2, 2))
seq.add(LRN())
seq.add(
Conv2D(input_channels=32,
num_filters=64,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
seq.add(BatchNormalization(input_shape=[h, w, 64]))
seq.add(RELU())
seq.add(MaxPooling(poolsize=(2, 2), stride=(2, 2), padding='SAME'))
h, w = same(in_height=h, in_width=w, stride=(2, 2), kernel_size=(2, 2))
seq.add(LRN())
seq.add(Flatten())
seq.add(Linear(int(h * w * 64), 128))
seq.add(BatchNormalization(input_shape=[128]))
seq.add(Tanh())
seq.add(Dropout(0.8))
seq.add(Linear(128, 256))
seq.add(BatchNormalization(input_shape=[256]))
seq.add(Tanh())
seq.add(Dropout(0.8))
seq.add(Linear(256, 10))
seq.add(Softmax())
return seq
def __init__(self, num_iter):
self.num_iter = num_iter
self.layers = []
self.layers.append(
Conv2D(input_channels=4,
num_filters=8,
kernel_size=(5, 5),
stride=(1, 1),
padding='SAME'))
self.layers.append(
BatchNormalization(layer_type='conv', dim=8, short_memory=0.01))
self.layers.append(RELU())
self.layers.append(
Conv2D(input_channels=8,
num_filters=1,
kernel_size=(5, 5),
stride=(1, 1),
padding='SAME'))
self.layers.append(RELU())
def __init__(self, num_blocks):
self.num_blocks = num_blocks
self.blocks = []
for _ in range(self.num_blocks):
layers = []
layers.append(
Conv3D(input_channels=1,
num_filters=1,
kernel_size=(5, 5, 5),
stride=(1, 1, 1),
padding='SAME'))
layers.append(RELU())
# layers.append(BatchNormalization(layer_type='conv', dim=8, short_memory=0.01))
# layers.append(Conv3D(input_channels=16, num_filters=16, kernel_size=(5,5,5), stride=(1,1,1), padding='SAME'))
# layers.append(RELU())
# layers.append(BatchNormalization(layer_type='conv', dim=8, short_memory=0.01))
layers.append(
Conv3D(input_channels=1,
num_filters=1,
kernel_size=(5, 5, 5),
stride=(1, 1, 1),
padding='SAME'))
layers.append(RELU())
# layers.append(BatchNormalization(layer_type='conv', dim=8, short_memory=0.01))
layers.append(
Conv3D(input_channels=1,
num_filters=1,
kernel_size=(5, 5, 5),
stride=(1, 1, 1),
padding='SAME'))
layers.append(RELU())
self.blocks.append(layers)
self.blocks.append([
Conv3D(input_channels=1,
num_filters=1,
kernel_size=(3, 3, 3),
stride=(1, 1, 1),
padding='SAME')
])
def __init__(self, nclass, h, w, c):
layers = []
template = TemplateModel(nclass, h, w, c)
layers.append(template)
layers.append(Flatten())
layers.append(Linear(template.output_dim, 200))
layers.append(RELU())
layers.append(Linear(200, nclass))
layers.append(Softmax())
self.startnode = tg.StartNode(input_vars=[None])
model_hn = tg.HiddenNode(prev=[self.startnode], layers=layers)
self.endnode = tg.EndNode(prev=[model_hn])
def __init__(self, input, num_blocks, kernel=(3, 3, 3)):
self.num_blocks = num_blocks
self.input = input
self.kernel = kernel
self.blocks = []
for _ in range(self.num_blocks):
layers = []
layers.append(
Conv3D(input_channels=self.input,
num_filters=self.input,
kernel_size=self.kernel,
stride=(1, 1, 1),
padding='SAME'))
layers.append(RELU())
layers.append(
Conv3D(input_channels=self.input,
num_filters=self.input,
kernel_size=self.kernel,
stride=(1, 1, 1),
padding='SAME'))
self.blocks.append(layers)
def __init__(self, input, BN_name, kernel=(3, 3, 3), iterate=1):
self.layers = []
self.int_ = 0
self.input = input
self.kernel = kernel
self.iterate = iterate
self.layers.append(
Conv3D(input_channels=input,
num_filters=input,
kernel_size=kernel,
stride=(1, 1, 1),
padding='SAME'))
self.layers.append(TFBatchNormalization(BN_name + str(self.int_)))
self.int_ += 1
self.layers.append(RELU())
self.layers.append(
Conv3D(input_channels=input,
num_filters=input,
kernel_size=kernel,
stride=(1, 1, 1),
padding='SAME'))
self.layers.append(TFBatchNormalization(BN_name + str(self.int_)))
self.int_ += 1
def Vanilla_Classifier(X_train, y_train, X_valid, y_valid, restore):
batchsize = 100
learning_rate = 0.001
_, h, w, c = X_train.shape
_, nclass = y_train.shape
g = tf.Graph()
with g.as_default():
data_train = tg.SequentialIterator(X_train, y_train, batchsize=batchsize)
data_valid = tg.SequentialIterator(X_valid, y_valid, batchsize=batchsize)
X_ph = tf.placeholder('float32', [None, h, w, c])
# y_ph = tf.placeholder('float32', [None, nclass])
y_phs = []
for comp in [nclass]:
y_phs.append(tf.placeholder('float32', [None, comp]))
dim = int(h*w*c)
scope = 'encoder'
start = tg.StartNode(input_vars=[X_ph])
h1_Node = tg.HiddenNode(prev=[start],
layers=[Sigmoid(),
TFBatchNormalization(name= scope + '/vanilla1'),
RELU(),
Flatten(),
Sigmoid(),
TFBatchNormalization(name=scope + '/vanilla2')])
h2_Node = tg.HiddenNode(prev=[h1_Node],
layers=[Linear(prev_dim=dim, this_dim=nclass),
Softmax()])
end_nodes = [tg.EndNode(prev=[h2_Node])]
graph = Graph(start=[start], end=end_nodes)
train_outs_sb = graph.train_fprop()
test_outs = graph.test_fprop()
ttl_mse = []
# import pdb; pdb.set_trace()
for y_ph, out in zip(y_phs, train_outs_sb):
#ttl_mse.append(tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_ph, out)))
ttl_mse.append(tf.reduce_mean((y_ph-out)**2))
mse = sum(ttl_mse)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(mse)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(mse)
saver = tf.train.Saver()
vardir = './var/2'
if not os.path.exists(vardir):
os.makedirs(vardir)
gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
tf.set_random_seed(1)
init = tf.global_variables_initializer()
with tf.Session(config = tf.ConfigProto(gpu_options = gpu_options)) as sess:
# print '=======session start'
sess.run(init)
if restore == 1:
re_saver = tf.train.Saver()
re_saver.restore(sess, vardir + "/model.ckpt")
print("Model restored.")
max_epoch = 100
temp_acc = []
for epoch in range(max_epoch):
train_error = 0
train_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_train:
feed_dict = {X_ph:X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
sess.run(optimizer, feed_dict=feed_dict)
train_outs = sess.run(train_outs_sb, feed_dict=feed_dict)
train_error += total_mse(train_outs, [ys])[0]
train_accuracy += total_accuracy(train_outs, [ys])[0]
ttl_examples += len(X_batch)
valid_error = 0
valid_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_valid:
feed_dict = {X_ph:X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
valid_outs = sess.run(test_outs, feed_dict=feed_dict)
valid_error += total_mse(valid_outs, [ys])[0]
valid_accuracy += total_accuracy(valid_outs, [ys])[0]
ttl_examples += len(X_batch)
save_path = saver.save(sess, vardir + "/model.ckpt")
# print("Model saved in file: %s" % save_path)
temp_acc.append(valid_accuracy/float(ttl_examples))
print 'max accuracy is:\t', max(temp_acc)
def Encoder_Classifier(X_train, y_train, X_valid, y_valid, restore):
batchsize = 100
learning_rate = 0.001
_, h, w, c = X_train.shape
_, nclass = y_train.shape
g = tf.Graph()
with g.as_default():
data_train = tg.SequentialIterator(X_train, y_train, batchsize=batchsize)
data_valid = tg.SequentialIterator(X_valid, y_valid, batchsize=batchsize)
X_ph = tf.placeholder('float32', [None, h, w, c])
y_phs = []
for comp in [nclass]:
y_phs.append(tf.placeholder('float32', [None, comp]))
start = tg.StartNode(input_vars=[X_ph])
h1, w1 = valid(h, w, filters=(5,5), strides=(1,1))
h2, w2 = valid(h1, w1, filters=(5,5), strides=(2,2))
h3, w3 = valid(h2, w2, filters=(5,5), strides=(2,2))
flat_dim = int(h3*w3*32)
scope = 'encoder'
bottleneck_dim = 300
enc_hn = tg.HiddenNode(prev=[start],
layers=[Conv2D(input_channels=c, num_filters=32, kernel_size=(5,5), stride=(1,1), padding='VALID'),
TFBatchNormalization(name=scope + '/genc1'),
RELU(),
Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
TFBatchNormalization(name=scope + '/genc2'),
RELU(),
Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
TFBatchNormalization(name=scope + '/genc3'),
RELU(),
Flatten(),
Linear(flat_dim, 300),
TFBatchNormalization(name=scope + '/genc4'),
RELU(),
Linear(300, bottleneck_dim),
Tanh()
])
h2_Node = tg.HiddenNode(prev=[enc_hn],
layers=[Linear(prev_dim=bottleneck_dim, this_dim=nclass),
Softmax()])
end_nodes = [tg.EndNode(prev=[h2_Node])]
graph = Graph(start=[start], end=end_nodes)
train_outs_sb = graph.train_fprop()
test_outs = graph.test_fprop()
ttl_mse = []
# import pdb; pdb.set_trace()
for y_ph, out in zip(y_phs, train_outs_sb):
#ttl_mse.append(tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_ph, out)))
ttl_mse.append(tf.reduce_mean((y_ph-out)**2))
mse = sum(ttl_mse)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(mse)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(mse)
gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
# saver_init = tf.train.Saver()
saver = tf.train.Saver()
vardir = './var/1'
if not os.path.exists(vardir):
os.makedirs(vardir)
tf.set_random_seed(1)
init = tf.global_variables_initializer()
with tf.Session(config = tf.ConfigProto(gpu_options = gpu_options)) as sess:
sess.run(init)
if restore == 1:
re_saver = tf.train.Saver()
re_saver.restore(sess, vardir + "/model.ckpt")
print("Model restored.")
# save_path = saver_init.save(sess, vardir + "/init.ckpt")
# print("Model saved in file: %s" % save_path)
max_epoch = 2
temp_acc = []
for epoch in range(max_epoch):
# print 'epoch:', epoch
train_error = 0
train_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_train:
feed_dict = {X_ph:X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
# import pdb; pdb.set_trace()
sess.run(optimizer, feed_dict=feed_dict)
train_outs = sess.run(train_outs_sb, feed_dict=feed_dict)
train_error += total_mse(train_outs, [ys])[0]
train_accuracy += total_accuracy(train_outs, [ys])[0]
ttl_examples += len(X_batch)
valid_error = 0
valid_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_valid:
feed_dict = {X_ph:X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
valid_outs = sess.run(test_outs, feed_dict=feed_dict)
valid_error += total_mse(valid_outs, [ys])[0]
valid_accuracy += total_accuracy(valid_outs, [ys])[0]
ttl_examples += len(X_batch)
temp_acc.append(valid_accuracy/float(ttl_examples))
save_path = saver.save(sess, vardir + "/model.ckpt")
print("Model saved in file: %s" % save_path)
print 'max accuracy is:\t', max(temp_acc)
def __init__(self, nclass, h, w, c):
layers = []
layers.append(
Conv2D(input_channels=c,
num_filters=96,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(BatchNormalization(input_shape=[h, w, 96]))
layers.append(
Conv2D(input_channels=96,
num_filters=96,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(Dropout(0.5))
layers.append(
Conv2D(input_channels=96,
num_filters=96,
kernel_size=(3, 3),
stride=(2, 2),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(2, 2), kernel_size=(3, 3))
layers.append(BatchNormalization(input_shape=[h, w, 96]))
layers.append(
Conv2D(input_channels=96,
num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(Dropout(0.5))
layers.append(
Conv2D(input_channels=192,
num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(BatchNormalization(input_shape=[h, w, 192]))
layers.append(
Conv2D(input_channels=192,
num_filters=192,
kernel_size=(3, 3),
stride=(2, 2),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(2, 2), kernel_size=(3, 3))
layers.append(Dropout(0.5))
layers.append(
Conv2D(input_channels=192,
num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(3, 3))
layers.append(BatchNormalization(input_shape=[h, w, 192]))
layers.append(
Conv2D(input_channels=192,
num_filters=192,
kernel_size=(1, 1),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(1, 1))
layers.append(Dropout(0.5))
layers.append(
Conv2D(input_channels=192,
num_filters=nclass,
kernel_size=(1, 1),
stride=(1, 1),
padding='SAME'))
layers.append(RELU())
h, w = same(in_height=h, in_width=w, stride=(1, 1), kernel_size=(1, 1))
layers.append(BatchNormalization(input_shape=[h, w, nclass]))
layers.append(
AvgPooling(poolsize=(h, w), stride=(1, 1), padding='VALID'))
layers.append(Flatten())
layers.append(Softmax())
self.startnode = tg.StartNode(input_vars=[None])
model_hn = tg.HiddenNode(prev=[self.startnode], layers=layers)
self.endnode = tg.EndNode(prev=[model_hn])
def model_Inception_Resnet(img=(84, 256, 256)):
with tf.name_scope('WMH'):
seq = tg.Sequential()
convStride = (1, 1, 1)
poolStride = (2, 2, 2)
kernelSize = (3, 3, 3)
seq.add(
Conv3D(input_channels=1,
num_filters=8,
kernel_size=(5, 5, 5),
stride=convStride,
padding='SAME'))
#seq.add(TFBatchNormalization(name='b1'))
seq.add(
MaxPool3D(poolsize=(2, 2, 2), stride=poolStride, padding='SAME'))
layerSize1 = updateConvLayerSize(img, poolStride)
seq.add(RELU())
seq.add(InceptionResnet_3D(8, type='v2_out8'))
seq.add(
Conv3D(input_channels=8,
num_filters=16,
kernel_size=kernelSize,
stride=convStride,
padding='SAME'))
#seq.add(TFBatchNormalization(name='b2'))
seq.add(
MaxPool3D(poolsize=(2, 2, 2), stride=poolStride, padding='SAME'))
layerSize2 = updateConvLayerSize(layerSize1, poolStride)
seq.add(RELU())
seq.add(InceptionResnet_3D(16, type='v1_out16'))
seq.add(
Conv3D(input_channels=16,
num_filters=16,
kernel_size=kernelSize,
stride=convStride,
padding='SAME'))
seq.add(RELU())
seq.add(
Conv3D(input_channels=16,
num_filters=32,
kernel_size=kernelSize,
stride=convStride,
padding='SAME'))
#seq.add(TFBatchNormalization(name='b3'))
seq.add(
MaxPool3D(poolsize=(2, 2, 2), stride=poolStride, padding='SAME'))
#layerSize3 = updateConvLayerSize(layerSize2,poolStride)
seq.add(RELU())
seq.add(InceptionResnet_3D(32, type='v1_out16'))
seq.add(
Conv3D_Tranpose1(input_channels=16,
num_filters=16,
output_shape=layerSize2,
kernel_size=kernelSize,
stride=poolStride,
padding='SAME'))
seq.add(RELU())
seq.add(InceptionResnet_3D(16, type='v1_out16'))
seq.add(
Conv3D(input_channels=16,
num_filters=16,
kernel_size=kernelSize,
stride=convStride,
padding='SAME'))
seq.add(RELU())
seq.add(
Conv3D_Tranpose1(input_channels=16,
num_filters=8,
output_shape=layerSize1,
kernel_size=kernelSize,
stride=poolStride,
padding='SAME'))
seq.add(RELU())
seq.add(InceptionResnet_3D(8, type='v2_out8'))
seq.add(
Conv3D(input_channels=8,
num_filters=8,
kernel_size=kernelSize,
stride=convStride,
padding='SAME'))
seq.add(RELU())
# num_filter=3 --> Background, WhiteMatter, Others
seq.add(
Conv3D_Tranpose1(input_channels=8,
num_filters=3,
output_shape=img,
kernel_size=kernelSize,
stride=poolStride,
padding='SAME'))
##
seq.add(RELU())
seq.add(
Conv3D(input_channels=3,
num_filters=3,
kernel_size=(1, 1, 1),
stride=convStride,
padding='SAME'))
##
seq.add(Softmax())
return seq
def __init__(self, h, w, c, z_dim=100, gf_dim=64, df_dim=64):
self.z_dim = z_dim
out_shape2 = same_nd([h, w], kernel_size=(5, 5), stride=(2, 2))
out_shape4 = same_nd(out_shape2, kernel_size=(5, 5), stride=(2, 2))
out_shape8 = same_nd(out_shape4, kernel_size=(5, 5), stride=(2, 2))
out_shape16 = same_nd(out_shape8, kernel_size=(5, 5), stride=(2, 2))
h16, w16 = out_shape16
with tf.variable_scope('Generator'):
self.g_layers = [
Linear(z_dim, 8 * gf_dim * h16 * w16),
Reshape([-1, h16, w16, 8 * gf_dim]),
# TFBatchNormalization(name='gbn1'),
BatchNormalization(input_shape=[h16, w16, 8 * gf_dim]),
RELU(),
Conv2D_Transpose(input_channels=8 * gf_dim,
num_filters=4 * gf_dim,
output_shape=out_shape8,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
# TFBatchNormalization(name='gbn2'),
BatchNormalization(input_shape=out_shape8 + [4 * gf_dim]),
RELU(),
Conv2D_Transpose(input_channels=4 * gf_dim,
num_filters=2 * gf_dim,
output_shape=out_shape4,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
# TFBatchNormalization(name='gbn3'),
BatchNormalization(input_shape=out_shape4 + [2 * gf_dim]),
RELU(),
Conv2D_Transpose(input_channels=2 * gf_dim,
num_filters=gf_dim,
output_shape=out_shape2,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
# TFBatchNormalization(name='gbn4'),
BatchNormalization(input_shape=out_shape2 + [gf_dim]),
RELU(),
Conv2D_Transpose(input_channels=gf_dim,
num_filters=c,
output_shape=(h, w),
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
# Sigmoid()
]
out_shape2 = same_nd([h, w], kernel_size=(5, 5), stride=(2, 2))
out_shape4 = same_nd(out_shape2, kernel_size=(5, 5), stride=(2, 2))
out_shape8 = same_nd(out_shape4, kernel_size=(5, 5), stride=(2, 2))
out_shape16 = same_nd(out_shape8, kernel_size=(5, 5), stride=(2, 2))
h16, w16 = out_shape16
with tf.variable_scope('Discriminator'):
self.d1_layers = [
Conv2D(input_channels=c,
num_filters=df_dim,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
LeakyRELU(),
Conv2D(input_channels=df_dim,
num_filters=2 * df_dim,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
]
# TFBatchNormalization(name='dbn1'),
self.d2_layers = [
BatchNormalization(input_shape=out_shape4 + [2 * df_dim]),
LeakyRELU(),
Conv2D(input_channels=2 * df_dim,
num_filters=4 * df_dim,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
]
self.d3_layers = [
# TFBatchNormalization(name='dbn2'),
BatchNormalization(input_shape=out_shape8 + [4 * df_dim]),
LeakyRELU(),
Conv2D(input_channels=4 * df_dim,
num_filters=8 * df_dim,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'),
]
self.d4_layers = [
# TFBatchNormalization(name='dbn3'),
BatchNormalization(input_shape=out_shape16 + [8 * df_dim]),
LeakyRELU(),
ReduceMax(reduction_indices=[1, 2]),
]
self.d5_layers = [
Flatten(),
Linear(8 * df_dim, 1),
# LeakyRELU(),
# Linear(1000, 1)
# Sigmoid()
]
print('====:', 8 * df_dim)
def model3D_2(img=(84, 256, 256)):
with tf.name_scope('WMH_2Chan_Input'):
seq = tg.Sequential()
convStride = (1, 1, 1)
poolStride = (2, 2, 2)
seq.add(
Conv3D(input_channels=1,
num_filters=8,
kernel_size=(5, 5, 5),
stride=convStride,
padding='SAME'))
seq.add(TFBatchNormalization(name='b1'))
seq.add(
MaxPool3D(poolsize=(2, 2, 2), stride=poolStride, padding='SAME'))
layerSize1 = updateConvLayerSize(img, poolStride)
seq.add(RELU())
seq.add(
Conv3D(input_channels=8,
num_filters=16,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
seq.add(TFBatchNormalization(name='b2'))
## Extra MaxPool
#seq.add(MaxPool3D(poolsize=(2,2,2), stride=poolStride, padding='SAME'))
#layerSize2 = updateConvLayerSize(layerSize1,convStride)
#seq.add(RELU())
## Extra Conv
#seq.add(Conv3D(input_channels=16, num_filters=16, kernel_size=(3,3,3), stride=convStride, padding='SAME'))
#seq.add(TFBatchNormalization(name='b3'))
seq.add(
Conv3D_Tranpose1(input_channels=16,
num_filters=8,
output_shape=layerSize1,
kernel_size=(3, 3, 3),
stride=convStride,
padding='SAME'))
seq.add(TFBatchNormalization(name='b4'))
seq.add(RELU())
seq.add(
Conv3D_Tranpose1(input_channels=8,
num_filters=2,
output_shape=img,
kernel_size=(3, 3, 3),
stride=poolStride,
padding='SAME'))
#seq.add(TFBatchNormalization(name='b5'))
#seq.add(RELU())
#seq.add(Conv3D(input_channels=2, num_filters=2, kernel_size=(1,1,1), stride=convStride, padding='SAME'))
##
##
#seq.add(RELU())
#seq.add(Conv3D(input_channels=2, num_filters=2, kernel_size=(3,3,3), stride=convStride, padding='SAME'))
# seq.add(RELU())
# seq.add(Conv3D(input_channels=3, num_filters=3, kernel_size=(3,3,3), stride=convStride, padding='SAME'))
# seq.add(RELU())
# seq.add(Conv3D(input_channels=3, num_filters=3, kernel_size=(3,3,3), stride=convStride, padding='SAME'))
#
##
seq.add(Softmax())
#seq.add(Sigmoid())
return seq
def CNN_Classifier(X_train, y_train, X_valid, y_valid, restore):
batchsize = 64
learning_rate = 0.001
_, h, w, c = X_train.shape
_, nclass = y_train.shape
g = tf.Graph()
with g.as_default():
data_train = tg.SequentialIterator(X_train,
y_train,
batchsize=batchsize)
data_valid = tg.SequentialIterator(X_valid,
y_valid,
batchsize=batchsize)
X_ph = tf.placeholder('float32', [None, h, w, c])
y_phs = []
for comp in [nclass]:
y_phs.append(tf.placeholder('float32', [None, comp]))
start = tg.StartNode(input_vars=[X_ph])
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(1, 1), filters=(2, 2))
h, w = valid(in_height=h, in_width=w, strides=(2, 2), filters=(2, 2))
# import pdb; pdb.set_trace()
#h1, w1 = valid(ch_embed_dim, word_len, strides=(1,1), filters=(ch_embed_dim,4))
num = 32
dim = int(h * w * num)
h1_Node = tg.HiddenNode(prev=[start],
layers=[
Conv2D(input_channels=c,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer1'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer2'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer3'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer4'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer5'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer6'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer7'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer8'),
RELU(),
Dropout(dropout_below=0.5),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer9'),
RELU(),
Conv2D(input_channels=num,
num_filters=num,
padding='VALID',
kernel_size=(2, 2),
stride=(1, 1)),
TFBatchNormalization(name='layer10'),
RELU(),
MaxPooling(poolsize=(2, 2),
stride=(2, 2),
padding='VALID'),
Reshape(shape=(-1, dim))
])
h2_Node = tg.HiddenNode(
prev=[h1_Node],
layers=[Linear(prev_dim=dim, this_dim=nclass),
Softmax()])
end_nodes = [tg.EndNode(prev=[h2_Node])]
graph = Graph(start=[start], end=end_nodes)
train_outs_sb = graph.train_fprop()
test_outs = graph.test_fprop()
ttl_mse = []
# import pdb; pdb.set_trace()
for y_ph, out in zip(y_phs, train_outs_sb):
#ttl_mse.append(tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_ph, out)))
ttl_mse.append(tf.reduce_mean((y_ph - out)**2))
mse = sum(ttl_mse)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(mse)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(mse)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
saver = tf.train.Saver()
vardir = './var/5'
if not os.path.exists(vardir):
os.makedirs(vardir)
tf.set_random_seed(1)
init = tf.global_variables_initializer()
with tf.Session(config=tf.ConfigProto(
gpu_options=gpu_options)) as sess:
sess.run(init)
if restore == 1:
re_saver = tf.train.Saver()
re_saver.restore(sess, vardir + "/model.ckpt")
print("Model restored.")
max_epoch = 100
temp_acc = []
for epoch in range(max_epoch):
# print 'epoch:', epoch
train_error = 0
train_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_train:
feed_dict = {X_ph: X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
# import pdb; pdb.set_trace()
sess.run(optimizer, feed_dict=feed_dict)
train_outs = sess.run(train_outs_sb, feed_dict=feed_dict)
train_error += total_mse(train_outs, [ys])[0]
train_accuracy += total_accuracy(train_outs, [ys])[0]
ttl_examples += len(X_batch)
valid_error = 0
valid_accuracy = 0
ttl_examples = 0
for X_batch, ys in data_valid:
feed_dict = {X_ph: X_batch}
for y_ph, y_batch in zip(y_phs, [ys]):
feed_dict[y_ph] = y_batch
valid_outs = sess.run(test_outs, feed_dict=feed_dict)
valid_error += total_mse(valid_outs, [ys])[0]
valid_accuracy += total_accuracy(valid_outs, [ys])[0]
ttl_examples += len(X_batch)
temp_acc.append(valid_accuracy / float(ttl_examples))
save_path = saver.save(sess, vardir + "/model.ckpt")
print("Model saved in file: %s" % save_path)
print 'max accuracy is:\t', max(temp_acc)
def model(nclass, h, w, c):
with tf.name_scope('Cifar10AllCNN'):
seq = tg.Sequential()
seq.add(
Conv2D(num_filters=96,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(BatchNormalization())
seq.add(
Conv2D(num_filters=96,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(Dropout(0.5))
seq.add(
Conv2D(num_filters=96,
kernel_size=(3, 3),
stride=(2, 2),
padding='SAME'))
seq.add(RELU())
seq.add(BatchNormalization())
seq.add(
Conv2D(num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(Dropout(0.5))
seq.add(
Conv2D(num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(BatchNormalization())
seq.add(
Conv2D(num_filters=192,
kernel_size=(3, 3),
stride=(2, 2),
padding='SAME'))
seq.add(RELU())
seq.add(Dropout(0.5))
seq.add(
Conv2D(num_filters=192,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(BatchNormalization())
seq.add(
Conv2D(num_filters=192,
kernel_size=(1, 1),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(Dropout(0.5))
seq.add(
Conv2D(num_filters=nclass,
kernel_size=(1, 1),
stride=(1, 1),
padding='SAME'))
seq.add(RELU())
seq.add(BatchNormalization())
seq.add(AvgPooling(poolsize=(8, 8), stride=(1, 1), padding='VALID'))
seq.add(Flatten())
seq.add(Softmax())
return seq
def generator(self):
self.generator_called = True
with self.tf_graph.as_default():
scope = 'Generator'
with tf.name_scope(scope):
h1, w1 = valid(self.h,
self.w,
kernel_size=(5, 5),
stride=(1, 1))
h2, w2 = valid(h1, w1, kernel_size=(5, 5), stride=(2, 2))
h3, w3 = valid(h2, w2, kernel_size=(5, 5), stride=(2, 2))
flat_dim = int(h3 * w3 * 32)
print('h1:{}, w1:{}'.format(h1, w1))
print('h2:{}, w2:{}'.format(h2, w2))
print('h3:{}, w3:{}'.format(h3, w3))
print('flat dim:{}'.format(flat_dim))
self.gen_real_sn = tg.StartNode(input_vars=[self.real_ph])
enc_hn = tg.HiddenNode(
prev=[self.gen_real_sn],
layers=[
Conv2D(input_channels=self.c,
num_filters=32,
kernel_size=(5, 5),
stride=(1, 1),
padding='VALID'),
TFBatchNormalization(name=scope + '/genc1'),
RELU(),
Conv2D(input_channels=32,
num_filters=32,
kernel_size=(5, 5),
stride=(2, 2),
padding='VALID'),
TFBatchNormalization(name=scope + '/genc2'),
RELU(),
Conv2D(input_channels=32,
num_filters=32,
kernel_size=(5, 5),
stride=(2, 2),
padding='VALID'),
TFBatchNormalization(name=scope + '/genc3'),
RELU(),
# Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
# RELU(),
Flatten(),
Linear(flat_dim, 300),
TFBatchNormalization(name=scope + '/genc4'),
RELU(),
Linear(300, self.bottleneck_dim),
Tanh(),
])
self.noise_sn = tg.StartNode(input_vars=[self.noise_ph])
self.gen_hn = tg.HiddenNode(
prev=[self.noise_sn, enc_hn],
input_merge_mode=Sum(),
layers=[
Linear(self.bottleneck_dim, flat_dim),
RELU(),
######[ Method 0 ]######
# Reshape((-1, h3, w3, 32)),
# Conv2D_Transpose(input_channels=32, num_filters=100, output_shape=(h2,w2),
# kernel_size=(5,5), stride=(2,2), padding='VALID'),
######[ End Method 0 ]######
######[ Method 1 ]######
Reshape((-1, 1, 1, flat_dim)),
# Reshape((-1, h))
Conv2D_Transpose(input_channels=flat_dim,
num_filters=200,
output_shape=(h3, w3),
kernel_size=(h3, w3),
stride=(1, 1),
padding='VALID'),
# BatchNormalization(layer_type='conv', dim=200, short_memory=0.01),
TFBatchNormalization(name=scope + '/g1'),
RELU(),
Conv2D_Transpose(input_channels=200,
num_filters=100,
output_shape=(h2, w2),
kernel_size=(5, 5),
stride=(2, 2),
padding='VALID'),
# BatchNormalization(layer_type='conv', dim=100, short_memory=0.01),
######[ End Method 1 ]######
TFBatchNormalization(name=scope + '/g2'),
RELU(),
Conv2D_Transpose(input_channels=100,
num_filters=50,
output_shape=(h1, w1),
kernel_size=(5, 5),
stride=(2, 2),
padding='VALID'),
# BatchNormalization(layer_type='conv', dim=50, short_memory=0.01),
TFBatchNormalization(name=scope + '/g3'),
RELU(),
Conv2D_Transpose(input_channels=50,
num_filters=self.c,
output_shape=(self.h, self.w),
kernel_size=(5, 5),
stride=(1, 1),
padding='VALID'),
SetShape((-1, self.h, self.w, self.c)),
Sigmoid()
])
h, w = valid(self.h, self.w, kernel_size=(5, 5), stride=(1, 1))
h, w = valid(h, w, kernel_size=(5, 5), stride=(2, 2))
h, w = valid(h, w, kernel_size=(5, 5), stride=(2, 2))
h, w = valid(h, w, kernel_size=(h3, w3), stride=(1, 1))
y_en = tg.EndNode(prev=[self.gen_hn])
graph = tg.Graph(start=[self.noise_sn, self.gen_real_sn],
end=[y_en])
G_train_sb = graph.train_fprop()[0]
G_test_sb = graph.test_fprop()[0]
gen_var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=scope)
return self.y_ph, self.noise_ph, G_train_sb, G_test_sb, gen_var_list
def Residual_UNET(input, img=(84, 256, 256)):
with tf.name_scope('WMH'):
convStride = (1, 1, 1)
poolStride = (2, 2, 2)
kSize3 = (3, 3, 3)
#kSize5 = (5,5,5)
#x_dim = 50
#component_dim = 100
#batchsize = 32
#learning_rate = 0.01
#x_ph = tf.placeholder('float32', [None, x_dim])
#start = StartNode(input_vars=[x_ph])
#h1 = HiddenNode(prev=[start], layers=[Linear(x_dim, component_dim), Softmax()])
#e1 = EndNode(prev=[h1], input_merge_mode=Sum())
#e3 = EndNode(prev=[h1, h2, h3], input_merge_mode=Sum())
start = StartNode(input_vars=[input])
Layer01 = [
Conv3D(input_channels=1,
num_filters=8,
kernel_size=kSize3,
stride=convStride,
padding='SAME')
]
#Layer01.append(RELU())
LayerPool = MaxPool3D(poolsize=(2, 2, 2),
stride=poolStride,
padding='SAME')
Layer02 = [LayerPool]
#Layer02.append(Testing(1))
Layer02.append(
Conv3D(input_channels=8,
num_filters=16,
kernel_size=kSize3,
stride=convStride,
padding='SAME'))
#Layer02.append(RELU())
#Layer02.append(Testing(2))
Layer02.append(ResidualBlock3D(16, 'L02'))
#Layer02.append(Testing(3))
layerSize1 = updateConvLayerSize(img, poolStride)
Layer03 = [LayerPool]
Layer03.append(
Conv3D(input_channels=16,
num_filters=32,
kernel_size=kSize3,
stride=convStride,
padding='SAME'))
#Layer03.append(RELU())
Layer03.append(ResidualBlock3D(32, 'L03'))
#layerSize2 = updateConvLayerSize(layerSize1,poolStride)
Layer03.append(
Conv3D_Tranpose1(input_channels=32,
num_filters=16,
output_shape=layerSize1,
kernel_size=kSize3,
stride=poolStride,
padding='SAME'))
#Layer03.append(RELU())
#Layer04 = [Conv3D(input_channels=32, num_filters=64, kernel_size=kSize5, stride=convStride, padding='SAME')]
#Layer04.append(ResidualBlock3D(64,'L03'))
conv8 = HiddenNode(prev=[start], layers=Layer01)
resBlock16 = HiddenNode(prev=[conv8], layers=Layer02)
resBlock32_16 = HiddenNode(prev=[resBlock16], layers=Layer03)
residualLong16 = HiddenNode(prev=[resBlock32_16, resBlock16],
input_merge_mode=Sum())
Layer04 = [ResidualBlock3D(16, 'L04')]
Layer04.append(
Conv3D_Tranpose1(input_channels=16,
num_filters=8,
output_shape=img,
kernel_size=kSize3,
stride=poolStride,
padding='SAME'))
Layer04.append(RELU())
resBlock16_8 = HiddenNode(prev=[residualLong16], layers=Layer04)
residualLong8 = HiddenNode(prev=[resBlock16_8, conv8],
input_merge_mode=Sum())
Layer05 = [ResidualBlock3D(8, 'L05')]
Layer05.append(
Conv3D(input_channels=8,
num_filters=2,
kernel_size=kSize3,
stride=convStride,
padding='SAME'))
Layer05.append(Softmax())
resBlock8_2 = HiddenNode(prev=[residualLong8], layers=Layer05)
endNode = EndNode(prev=[resBlock8_2], input_merge_mode=NoChange())
graph = Graph(start=[start], end=[endNode])
#o1, o2, o3 = graph.train_fprop()
#o1_mse = tf.reduce_mean((y1_ph - o1)**2)
#o2_mse = tf.reduce_mean((y2_ph - o2)**2)
return graph
def train():
batchsize = 64
learning_rate = 0.001
max_epoch = 100
# batch x depth x height x width x channel
X_train = np.random.rand(1000, 20, 32, 32, 1)
M_train = np.random.rand(1000, 20, 32, 32, 1)
X_valid = np.random.rand(1000, 20, 32, 32, 1)
M_valid = np.random.rand(1000, 20, 32, 32, 1)
X_ph = tf.placeholder('float32', [None, 20, 32, 32, 1])
M_ph = tf.placeholder('float32', [None, 20, 32, 32, 1])
h, w = 32, 32
model = tg.Sequential()
# iter_model = tg.Sequential()
model.add(
Conv3D(input_channels=1,
num_filters=8,
kernel_size=(5, 5, 5),
stride=(1, 1, 1),
padding='SAME'))
model.add(RELU())
model.add(
Conv3D(input_channels=8,
num_filters=1,
kernel_size=(5, 5, 5),
stride=(1, 1, 1),
padding='SAME'))
# iter_model.add(RELU())
# model.add(Iterative(sequential=iter_model, num_iter=1))
model.add(Sigmoid())
M_train_s = model.train_fprop(X_ph)
M_valid_s = model.test_fprop(X_ph)
train_mse = tf.reduce_mean((M_ph - M_train_s)**2)
valid_mse = tf.reduce_mean((M_ph - M_valid_s)**2)
# train_mse = entropy(M_ph, M_train_s)
# valid_mse = entropy(M_ph, M_valid_s)
valid_f1 = image_f1(tf.to_int32(M_ph), tf.to_int32(M_valid_s >= 0.5))
data_train = tg.SequentialIterator(X_train, M_train, batchsize=batchsize)
data_valid = tg.SequentialIterator(X_valid, M_valid, batchsize=batchsize)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(train_mse)
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
for epoch in range(max_epoch):
print('epoch:', epoch)
print('..training')
pbar = ProgressBar(len(data_train))
n_exp = 0
for X_batch, M_batch in data_train:
pbar.update(n_exp)
sess.run(optimizer, feed_dict={X_ph: X_batch, M_ph: M_batch})
n_exp += len(X_batch)
print('..validating')
valid_f1_score, valid_mse_score = sess.run([valid_f1, valid_mse],
feed_dict={
X_ph: X_valid,
M_ph: M_valid
})
print('valid mse score:', valid_mse_score)
print('valid f1 score:', valid_f1_score)
def generator(self):
self.generator_called = True
with self.tf_graph.as_default():
scope = 'Generator'
with tf.name_scope(scope):
# X_ph = tf.placeholder('float32', [None, self.h, self.w, 1], name='X')
# X_sn = tg.StartNode(input_vars=[X_ph])
noise_ph = tf.placeholder('float32',
[None, self.bottleneck_dim],
name='noise')
self.noise_sn = tg.StartNode(input_vars=[noise_ph])
h1, w1 = valid(self.h,
self.w,
kernel_size=(5, 5),
stride=(1, 1))
h2, w2 = valid(h1, w1, kernel_size=(5, 5), stride=(2, 2))
h3, w3 = valid(h2, w2, kernel_size=(5, 5), stride=(2, 2))
flat_dim = int(h3 * w3 * 32)
print('h1:{}, w1:{}'.format(h1, w1))
print('h2:{}, w2:{}'.format(h2, w2))
print('h3:{}, w3:{}'.format(h3, w3))
print('flat dim:{}'.format(flat_dim))
# enc_hn = tg.HiddenNode(prev=[X_sn],
# layers=[Conv2D(input_channels=1, num_filters=32, kernel_size=(5,5), stride=(1,1), padding='VALID'),
# RELU(),
# Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
# RELU(),
# Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
# RELU(),
# # Conv2D(input_channels=32, num_filters=32, kernel_size=(5,5), stride=(2,2), padding='VALID'),
# # RELU(),
# Flatten(),
# Linear(flat_dim, 300),
# RELU(),
# # seq.add(Dropout(0.5))
# Linear(300, self.bottleneck_dim),
# Tanh(),
# ])
y_ph = tf.placeholder('float32', [None, self.nclass], name='y')
self.y_sn = tg.StartNode(input_vars=[y_ph])
noise_hn = tg.HiddenNode(prev=[self.noise_sn, self.y_sn],
input_merge_mode=Concat(1))
self.gen_hn = tg.HiddenNode(
prev=[noise_hn],
layers=[
Linear(self.bottleneck_dim + 10, flat_dim),
RELU(),
######[ Method 0 ]######
# Reshape((-1, h3, w3, 32)),
# Conv2D_Transpose(input_channels=32, num_filters=100, output_shape=(h2,w2),
# kernel_size=(5,5), stride=(2,2), padding='VALID'),
######[ End Method 0 ]######
######[ Method 1 ]######
Reshape((-1, 1, 1, flat_dim)),
Conv2D_Transpose(input_channels=flat_dim,
num_filters=200,
output_shape=(2, 2),
kernel_size=(2, 2),
stride=(1, 1),
padding='VALID'),
TFBatchNormalization(name=scope + '/dc1'),
RELU(),
Conv2D_Transpose(input_channels=200,
num_filters=100,
output_shape=(h2, w2),
kernel_size=(9, 9),
stride=(1, 1),
padding='VALID'),
######[ End Method 1 ]######
TFBatchNormalization(name=scope + '/dc2'),
RELU(),
Conv2D_Transpose(input_channels=100,
num_filters=50,
output_shape=(h1, w1),
kernel_size=(5, 5),
stride=(2, 2),
padding='VALID'),
TFBatchNormalization(name=scope + '/dc3'),
RELU(),
Conv2D_Transpose(input_channels=50,
num_filters=1,
output_shape=(self.h, self.w),
kernel_size=(5, 5),
stride=(1, 1),
padding='VALID'),
SetShape((-1, self.h, self.w, 1)),
Sigmoid()
])
y_en = tg.EndNode(prev=[self.gen_hn])
graph = tg.Graph(start=[self.noise_sn, self.y_sn], end=[y_en])
G_train_sb = graph.train_fprop()[0]
G_test_sb = graph.test_fprop()[0]
# import pdb; pdb.set_trace()
gen_var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=scope)
return y_ph, noise_ph, G_train_sb, G_test_sb, gen_var_list
def classifier(X_ph, X_gen_ph, h, w):
with tf.variable_scope('Classifier'):
X_sn = tg.StartNode(input_vars=[X_ph])
X_gen_sn = tg.StartNode(input_vars=[X_gen_ph])
h1, w1 = same(in_height=h,
in_width=w,
stride=(1, 1),
kernel_size=(3, 3))
h2, w2 = same(in_height=h1,
in_width=w1,
stride=(2, 2),
kernel_size=(2, 2))
h3, w3 = same(in_height=h2,
in_width=w2,
stride=(1, 1),
kernel_size=(3, 3))
h4, w4 = same(in_height=h3,
in_width=w3,
stride=(2, 2),
kernel_size=(2, 2))
print('---', h, w)
X_hn = tg.HiddenNode(prev=[X_sn],
layers=[
Conv2D(input_channels=1,
num_filters=32,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'),
BatchNormalization(input_shape=[h1, w1, 32]),
RELU(),
MaxPooling(poolsize=(2, 2),
stride=(2, 2),
padding='SAME'),
LRN(),
Conv2D(input_channels=32,
num_filters=64,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'),
BatchNormalization(input_shape=[h3, w3, 64]),
RELU(),
MaxPooling(poolsize=(2, 2),
stride=(2, 2),
padding='SAME'),
Flatten(),
])
X_gen_hn = tg.HiddenNode(
prev=[X_gen_sn],
layers=[
Conv2D(input_channels=1,
num_filters=32,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'),
BatchNormalization(input_shape=[h1, w1, 32]),
RELU(),
MaxPooling(poolsize=(2, 2), stride=(2, 2), padding='SAME'),
LRN(),
Conv2D(input_channels=32,
num_filters=64,
kernel_size=(3, 3),
stride=(1, 1),
padding='SAME'),
BatchNormalization(input_shape=[h3, w3, 64]),
RELU(),
MaxPooling(poolsize=(2, 2), stride=(2, 2), padding='SAME'),
Flatten(),
])
print('===', h4 * w4 * 64 * 2)
merge_hn = tg.HiddenNode(prev=[X_hn, X_gen_hn],
input_merge_mode=Concat(),
layers=[
Linear(h4 * w4 * 64 * 2, 100),
RELU(),
BatchNormalization(input_shape=[100]),
Linear(100, 1),
Sigmoid()
])
en = tg.EndNode(prev=[merge_hn])
graph = tg.Graph(start=[X_sn, X_gen_sn], end=[en])
y_train, = graph.train_fprop()
y_test, = graph.test_fprop()
return y_train, y_test
def train():
batchsize = 64
learning_rate = 0.001
max_epoch = 10
X_train = np.random.rand(1000, 32, 32, 3)
M_train = np.random.rand(1000, 32, 32, 1)
X_valid = np.random.rand(1000, 32, 32, 3)
M_valid = np.random.rand(1000, 32, 32, 1)
X_ph = tf.placeholder('float32', [None, 32, 32, 3])
M_ph = tf.placeholder('float32', [None, 32, 32, 1])
h, w = 32, 32
model = tg.Sequential()
model.add(
Conv2D(input_channels=3,
num_filters=8,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
h1, w1 = same(h, w, kernel_size=(5, 5), stride=(2, 2))
model.add(RELU())
model.add(
Conv2D(input_channels=8,
num_filters=16,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
h2, w2 = same(h1, w1, kernel_size=(5, 5), stride=(2, 2))
model.add(RELU())
model.add(
Conv2D_Transpose(input_channels=16,
num_filters=8,
output_shape=(h1, w1),
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
model.add(RELU())
model.add(
Conv2D_Transpose(input_channels=8,
num_filters=1,
output_shape=(h, w),
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
model.add(RELU())
iter_model = tg.Sequential()
iter_model.add(
Conv2D(input_channels=1,
num_filters=8,
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
iter_model.add(RELU())
iter_model.add(
Conv2D_Transpose(input_channels=8,
num_filters=1,
output_shape=(h, w),
kernel_size=(5, 5),
stride=(2, 2),
padding='SAME'))
model.add(Iterative(sequential=iter_model, num_iter=10))
M_train_s = model.train_fprop(X_ph)
M_valid_s = model.test_fprop(X_ph)
train_mse = tf.reduce_mean((M_ph - M_train_s)**2)
valid_mse = tf.reduce_mean((M_ph - M_valid_s)**2)
data_train = tg.SequentialIterator(X_train, M_train, batchsize=batchsize)
data_valid = tg.SequentialIterator(X_valid, M_valid, batchsize=batchsize)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(train_mse)
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
for epoch in range(max_epoch):
print('epoch:', epoch)
print('..training')
for X_batch, M_batch in data_train:
sess.run(optimizer, feed_dict={X_ph: X_batch, M_ph: M_batch})
print('..validating')
valid_mse_score = sess.run(valid_mse,
feed_dict={
X_ph: X_valid,
M_ph: M_valid
})
print('valid mse score:', valid_mse_score)
