def __init__(self, state_size, action_size):
self.states = tf.placeholder(tf.float32,
shape=[None, *state_size],
name="states")
self.labels = tf.placeholder(
tf.int32, shape=[None, 1],
name="labels") # size 1 because sparse loss is used.
# conv1 = layers.conv2d(self.states, filters=16, kernel_size=(8, 8), strides=(4, 4), activation='relu',
# name="conv1"),
# conv2 = layers.conv2d(conv1, filters=32, kernel_size=(4, 4), strides=(2, 2), activation='relu', name="conv2"),
# flatten = layers.flatten(conv2),
# dense = layers.dense(flatten, 256, activation='relu', name="features"),
#
# self.logits = layers.dense(dense, action_size, name="logits")
# self.value = layers.dense(dense, 1, name="values")
# conv1 = conv2d(self.states, filters=32, kernel_size=(3, 3), name='conv1')
with tf.variable_scope('layers'):
conv1 = layers.conv2d(self.states,
filters=32,
kernel_size=(3, 3),
activation='relu',
name='conv1')
conv2 = layers.conv2d(conv1,
filters=64,
kernel_size=(3, 3),
activation='relu',
name='conv2')
max_pool = layers.max_pooling2d(conv2, 2, 1, name='max_pool')
drop_1 = layers.dropout(max_pool, 0.25, name='drop1')
flatten = layers.flatten(drop_1, name='flatten')
dense = layers.dense(flatten, 128, activation='relu', name='dense')
drop2 = layers.dropout(dense, 0.5, name='drop2')
logits = layers.dense(drop2,
action_size,
activation='softmax',
name='logits')
self.output = tf.nn.softmax(logits, name='output')
# tf.one_hot(tf.arg_max(self.output, 1), depth=10)
print(tf.arg_max(self.output, 1))
self.test = tf.one_hot(tf.arg_max(self.output, 1), depth=10)
# input()
self.cost = tf.losses.sparse_softmax_cross_entropy(self.labels, logits)
self.acc, self.acc_op = tf.metrics.accuracy(self.labels,
tf.arg_max(self.output, 1))
# self.grad = tf.gradients(self.cost, self.states, stop_gradients=[self.states])
self.grad = tf.gradients(self.cost, tf.trainable_variables())
self.optimizer = tf.train.AdamOptimizer()
# print(tf.trainable_variables())
# print(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='layers'))
# print(self.grad)
self.apply_grad = self.optimizer.apply_gradients(
zip(
self.grad,
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='layers')))
def __init__(self, game, tensor_logs_dir):
self.game = game
self.shape = game.board_size
self.possible_moves_size = self.shape[1]
self.log_writer = tf.summary.FileWriter(tensor_logs_dir + "/losses")
self.checkpoint_dir = "checkpoints"
create_dir(self.checkpoint_dir)
self.graph = tf.Graph()
self.session = tf.Session(graph=self.graph)
self.saver = None
with tf.Session() as temp_session:
temp_session.run(tf.global_variables_initializer())
with self.graph.as_default():
self.input_boards = tf.placeholder(tf.float32, shape=[None, self.game.board.rows, self.game.board.cols])
self.dropout = tf.placeholder(tf.float32)
self.isTraining = tf.placeholder(tf.bool, name="is_training")
self.pi_losses_var = tf.Variable(0, dtype=tf.float32)
self.v_losses_var = tf.Variable(0, dtype=tf.float32)
X = tf.reshape(self.input_boards, [-1, self.shape[0], self.shape[1], 1])
h_conv1 = relu(batch_normalization(conv2d(X, 'same'), axis=3, training=self.isTraining))
h_conv2 = relu(batch_normalization(conv2d(h_conv1, 'same'), axis=3, training=self.isTraining))
h_conv3 = relu(batch_normalization(conv2d(h_conv2, 'valid'), axis=3, training=self.isTraining))
h_conv4 = relu(batch_normalization(conv2d(h_conv3, 'valid'), axis=3, training=self.isTraining))
h_conv4_flat = tf.reshape(h_conv4, [-1, config.num_channels * (self.shape[0] - 4) * (self.shape[1] - 4)])
s_fc1 = dropout(relu(batch_normalization(dense(h_conv4_flat, 1024), axis=1, training=self.isTraining)),
rate=self.dropout)
s_fc2 = dropout(relu(batch_normalization(dense(s_fc1, 512), axis=1, training=self.isTraining)),
rate=self.dropout)
self.pi = dense(s_fc2, self.possible_moves_size)
self.prob = tf.nn.softmax(self.pi)
self.v = tanh(dense(s_fc2, 1))
# Place holders for Predicted (pi, v)s
self.predicted_pis = tf.placeholder(dtype=tf.float32, shape=[None, self.possible_moves_size])
self.predicted_vs = tf.placeholder(dtype=tf.float32, shape=[None])
# Real Losses
self.loss_pi = tf.losses.softmax_cross_entropy(self.predicted_pis, self.pi)
self.loss_v = tf.losses.mean_squared_error(self.predicted_vs, tf.reshape(self.v, shape=[-1, ]))
self.total_loss = self.loss_pi + self.loss_v
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step = tf.train.AdamOptimizer(config.lr).minimize(self.total_loss)
self.session.run(tf.variables_initializer(self.graph.get_collection('variables')))
def _dense_block(self, x, nb_layers, nb_filter, grow_nb_filters=True, return_concat_list=False):
""" Build a dense_block where the output of each conv_block is fed to subsequent ones
Args:
x: tensor
nb_layers: the number of layers of conv_block to append to the model.
nb_filter: number of filters
grow_nb_filters: flag to decide to allow number of filters to grow
return_concat_list: return the list of feature maps along with the actual output
Returns:
tensor with nb_layers of conv_block appended
"""
x_list = [x]
for i in range(nb_layers):
with tf.variable_scope('denselayer_{}'.format(i), use_resource=True):
cb = self._conv_block(x, self.growth_rate)
x_list.append(cb)
x = concatenate([x, cb], self.axis)
if grow_nb_filters:
nb_filter += self.growth_rate
if self.dropout_rate:
x = dropout(x, self.dropout_rate, training=self.training)
if return_concat_list:
return x, nb_filter, x_list
else:
return x, nb_filter
def _x(ip):
x = batch_normalization(ip, **self.bn_kwargs)
x = tf.nn.relu(x)
if self.bottleneck:
inter_channel = nb_filter * 4
x = conv2d(x,
inter_channel, (1, 1),
kernel_initializer='he_normal',
padding='same',
use_bias=False,
**self.conv_kwargs)
x = batch_normalization(x, **self.bn_kwargs)
x = tf.nn.relu(x)
x = conv2d(x,
nb_filter, (3, 3),
kernel_initializer='he_normal',
padding='same',
use_bias=False,
**self.conv_kwargs)
if self.dropout_rate:
x = dropout(x, self.dropout_rate, training=self.training)
return x
def deconv_block(self, input, filters, conv, padding, scope):
with tf.variable_scope(scope):
deconv1 = conv2d_transpose(input,
filters,
kernel_size=(3, 3),
strides=[2, 2],
padding=padding)
deconv_shape = tf.shape(deconv1)
conv_shape = tf.shape(conv)
offsets = [
0, (conv_shape[1] - deconv_shape[1]) // 2,
(conv_shape[2] - deconv_shape[2]) // 2, 0
]
size = [-1, deconv_shape[1], deconv_shape[2], filters]
conv_crop = tf.slice(conv, offsets, size)
conv1 = tf.concat([deconv1, conv_crop], 3)
bn = batch_normalization(conv1)
drop = dropout(bn, .25)
conv2 = conv2d(drop,
filters,
kernel_size=(3, 3),
activation=tf.nn.relu,
name='middle1',
padding="SAME")
conv3 = conv2d(conv2,
filters,
kernel_size=(3, 3),
activation=tf.nn.relu,
name='middle2',
padding="SAME")
return conv3
def misconception_model(inputs,
filters_list,
kernel_size,
strides_list,
training,
objective_functions,
sub_filters=128,
sub_layers=2,
dropout_rate=0.5,
virtual_batch_size=None,
feature_means=None,
feature_stds=None):
""" A misconception tower.
Args:
input: a tensor of size [batch_size, 1, width, depth].
window_size: the width of the conv and pooling filters to apply.
depth: the depth of the output tensor.
levels: the height of the tower in misconception layers.
objective_functions: a list of objective functions to add to the top of
the network.
is_training: whether the network is training.
Returns:
a tensor of size [batch_size, num_classes].
"""
layers = []
net = inputs
if feature_means is not None:
net = net - tf.constant(feature_means)[None, None, :]
if feature_stds is not None:
net = net / (tf.constant(feature_stds) + 1e-6)
layers.append(net)
for filters, strides in zip(filters_list, strides_list):
net = misconception_with_bypass(net,
filters,
kernel_size,
strides,
training,
virtual_batch_size=virtual_batch_size)
layers.append(net)
outputs = []
for ofunc in objective_functions:
onet = net
for _ in range(sub_layers - 1):
onet = ly.conv1d(onet,
sub_filters,
1,
activation=None,
use_bias=False)
onet = ly.batch_normalization(
onet, training=training, virtual_batch_size=virtual_batch_size)
onet = tf.nn.relu(onet)
onet = ly.conv1d(onet, sub_filters, 1, activation=tf.nn.relu)
onet = ly.flatten(onet)
#
onet = ly.dropout(onet, training=training, rate=dropout_rate)
outputs.append(ofunc.build(onet))
return outputs, layers
def build_model(embs, lr_w, dropout_on=tf.constant(False),
min_cut=min_cut, max_cut=max_cut, cut_size=cut_size):
if embedding_dropout_type == 'none' or dropout_rate<0.01:
pass
elif embedding_dropout_type=='standard':
embs = tf.layaers.dropout(embs, rate=dropout_rate, training=dropout_on)
elif embedding_dropout_type=='field':
# random dropout some fields
embs = tf.layers.dropout(
embs, rate=dropout_rate, training=dropout_on,
noise_shape=[embs.shape[1], tf.constant(1)])
kwargs['scaling_factor'] = tf.cond(dropout_on, lambda: 1.-dropout_rate, lambda: 1.)
elif embedding_dropout_type=='embedding':
# random dropout some dimensions of embeddings
embs = tf.layers.dropout(
embs, rate=dropout_rate, training=dropout_on,
noise_shape=[tf.constant(1), embs.shape[2]])
kwargs['scaling_factor'] = tf.cond(dropout_on, lambda: 1.-dropout_rate, lambda: 1.)
else:
print("Unknown embedding_dropout_type! Embedding Dropout is bypassed.")
self.embs = embs
latent = get_latent(embs, lr_w, **kwargs)
latent = dropout(latent, dropout_rate, training=dropout_on)
logit = tf.layers.dense(latent, 1, name='last_fc', kernel_initializer=tf.initializers.he_uniform())
return logit, logit
def add_inference(self, inputs, training, nclass):
df = 'channels_first' if self.data_format == 'NCHW' else 'channels_last'
conv1 = layers.conv2d(inputs=inputs,
filters=32,
kernel_size=[5, 5],
padding="same",
data_format=df,
activation=tf.nn.relu)
pool1 = layers.max_pooling2d(inputs=conv1,
pool_size=[2, 2],
strides=2,
data_format=df)
conv2 = layers.conv2d(inputs=pool1,
filters=64,
kernel_size=[3, 3],
padding="same",
data_format=df,
activation=tf.nn.relu)
pool2 = layers.max_pooling2d(inputs=conv2,
pool_size=[2, 2],
strides=2,
data_format=df)
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
dense = layers.dense(inputs=pool2_flat,
units=1024,
activation=tf.nn.relu)
dropout = layers.dropout(inputs=dense, rate=0.4, training=training)
logits = layers.dense(inputs=dropout, units=nclass)
return logits
def build_discriminator(image, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse:
tf.get_variable_scope().reuse_variables()
#
t = image
t = conv2d(inputs=t,
filters=64,
kernel_size=[5, 5],
strides=2,
padding="same",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=128,
kernel_size=[5, 5],
strides=2,
padding="same",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = dropout(inputs=t, rate=DROP_RATE)
t = flatten(inputs=t)
t = dense(inputs=t, units=1, activation=tf.sigmoid)
decision = t
#print("\nD output shape: {}".format(decision.shape))
return decision
def mixed_7a(self, x):
with tf.variable_scope("mixed_7a"):
with tf.variable_scope("branch0"):
x0 = self.basic_conv2d(x,
256,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x0 = self.basic_conv2d(x0,
384,
kernel_size=3,
stride=2,
padding="same",
namescope="conv2",
use_bias=False)
with tf.variable_scope("branch1"):
x1 = self.basic_conv2d(x,
256,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x1 = self.basic_conv2d(x1,
288,
kernel_size=3,
stride=2,
padding="same",
namescope="conv2",
use_bias=False)
with tf.variable_scope("branch2"):
x2 = self.basic_conv2d(x,
256,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x2 = self.basic_conv2d(x2,
288,
kernel_size=3,
stride=1,
padding="same",
namescope="conv2",
use_bias=False)
x2 = self.basic_conv2d(x2,
320,
kernel_size=3,
stride=2,
padding="same",
namescope="conv3",
use_bias=False)
with tf.variable_scope("branch3"):
x3 = layers.max_pooling2d(x, 3, strides=2, padding="same")
x = tf.concat([x0, x1, x2, x3], axis=-1)
x = layers.dropout(x, noise_shape=[None, 1, 1, None])
return x
def mixed_5b(self, x):
with tf.variable_scope("mixed_5b"):
with tf.variable_scope("branch0"):
x0 = self.basic_conv2d(x,
96,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
with tf.variable_scope("branch1"):
x1 = self.basic_conv2d(x,
48,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x1 = self.basic_conv2d(x1,
64,
kernel_size=5,
stride=1,
padding="same",
namescope="conv2",
use_bias=False)
with tf.variable_scope("branch2"):
x2 = self.basic_conv2d(x,
64,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x2 = self.basic_conv2d(x2,
96,
kernel_size=3,
stride=1,
padding="same",
namescope="conv2",
use_bias=False)
x2 = self.basic_conv2d(x2,
96,
kernel_size=3,
stride=1,
padding="same",
namescope="conv3",
use_bias=False)
with tf.variable_scope("branch3"):
x3 = layers.average_pooling2d(x, 3, strides=1, padding="same")
x3 = self.basic_conv2d(x3,
64,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x = tf.concat([x0, x1, x2, x3], axis=-1)
x = layers.dropout(x, noise_shape=[None, 1, 1, None])
return x
def create_discriminator(self, img, rate, reuse=None):
# hard code all layer params
# 1. small kernels would lead to loss become 0 quickly
# try a very large one
# 2. More filters help the performance
# especially a lot more filters in generator
with tf.variable_scope("discriminator", reuse=reuse):
x = conv2d(img,
128, (5, 5),
strides=(2, 2),
padding='same',
data_format='channels_last',
dilation_rate=(1, 1),
activation=tf.nn.leaky_relu,
use_bias=True)
x = dropout(x, rate)
# assert tf.shape(x) == (64,64,1)
x = conv2d(x,
128, (5, 5),
strides=(2, 2),
padding='same',
data_format='channels_last',
dilation_rate=(1, 1),
activation=tf.nn.leaky_relu,
use_bias=True)
x = dropout(x, rate)
x = conv2d(x,
128, (5, 5),
strides=(1, 1),
padding='same',
data_format='channels_last',
dilation_rate=(1, 1),
activation=tf.nn.leaky_relu,
use_bias=True)
x = dropout(x, rate)
x = flatten(x)
x = fully_connected(x, 512, activation_fn=tf.nn.leaky_relu)
d_logits = fully_connected(x, 1, activation_fn=None)
d_prob = tf.sigmoid(d_logits)
return d_prob, d_logits
def create_generator(self, z, rate, is_training, reuse=None):
# hard code all layer params
upsample_power = 5 # 2**5=32
momentum = 0.99
with tf.variable_scope("generator", reuse=reuse):
# Since "mode collapse" issue, feature of FC needs to be small
# each layer come with dropout and batch_norm to improve the performance
x = fully_connected(z, 4 * 4 * 1, activation_fn=tf.nn.leaky_relu)
# x = dropout(x, rate = rate,training=is_training)
x = dropout(x, rate=rate)
x = tf.contrib.layers.batch_norm(x,
is_training=is_training,
decay=momentum)
x = tf.reshape(x, shape=(-1, 4, 4, 1))
for _ in range(upsample_power):
x = conv2d_transpose(x,
128, (5, 5),
strides=(2, 2),
padding='same',
data_format='channels_last',
activation=tf.nn.leaky_relu,
use_bias=True)
# x = dropout(x, rate = rate,training=is_training)
x = dropout(x, rate=rate)
x = tf.contrib.layers.batch_norm(x,
is_training=is_training,
decay=momentum)
x = conv2d_transpose(x,
1, (5, 5),
strides=(1, 1),
padding='same',
data_format='channels_last',
activation=tf.sigmoid,
use_bias=True)
# assert tf.shape(x) == (128,128,1)
# output is a matrix with -1 to 1
return x
def transpose_conv1d_layer(x,
n_out,
kernel_size,
stride=1,
activation=ACTIVATION,
regularize=True,
use_bias=True,
drop_rate=0.0,
batch_norm=BATCH_NORM,
training=True,
name=None,
reuse=None):
if batch_norm:
if name:
x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse)
else:
x = batch_norm_layer(x, training, name=name, reuse=reuse)
#wt_init = tf.truncated_normal_initializer(stddev=0.2)
#bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01)
wt_init = None
bi_init = None
if regularize:
wt_reg = WT_REG
bi_reg = BI_REG
else:
wt_reg = None
bi_reg = None
x_tmp = tf.expand_dims(x, 3)
x_tmp = tf.transpose(x_tmp, (0, 1, 3, 2))
y_tmp = layers.conv2d_transpose(x_tmp,
n_out,
kernel_size=[kernel_size, 1],
strides=(stride, 1),
padding='same',
data_format='channels_last',
activation=activation,
use_bias=add_bias,
kernel_initializer=wt_init,
bias_initializer=bi_init,
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name=None,
reuse=None)
y = tf.reduce_sum(y_tmp, axis=2)
y = layers.dropout(y, rate=drop_rate, training=training)
return y
def model(inputs, is_training, data_format, num_classes):
net = conv2d(inputs=inputs,
filters=96,
kernel_size=[7, 7],
strides=2,
padding='valid',
data_format=data_format,
activation=tf.nn.relu,
use_bias=True,
kernel_initializer=tf.variance_scaling_initializer(),
bias_initializer=tf.zeros_initializer())
net = max_pooling2d(inputs=net,
pool_size=[3, 3],
strides=2,
data_format=data_format)
net = fire_module(net, 16, 64, data_format)
net = fire_module(net, 16, 64, data_format)
net = fire_module(net, 32, 128, data_format)
net = max_pooling2d(inputs=net,
pool_size=[3, 3],
strides=2,
data_format=data_format)
net = fire_module(net, 32, 128, data_format)
net = fire_module(net, 48, 192, data_format)
net = fire_module(net, 48, 192, data_format)
net = fire_module(net, 64, 256, data_format)
net = max_pooling2d(inputs=net,
pool_size=[3, 3],
strides=2,
data_format=data_format)
net = fire_module(net, 64, 256, data_format)
net = dropout(inputs=net, rate=0.5, training=is_training)
net = conv2d(
inputs=net,
filters=num_classes,
kernel_size=[1, 1],
strides=1,
padding='valid', # no padding eqv. to pad=1 for 1x1 conv?
data_format=data_format,
activation=tf.nn.relu,
use_bias=True,
kernel_initializer=tf.initializers.random_normal(mean=0.0,
stddev=0.01),
bias_initializer=tf.zeros_initializer())
net = average_pooling2d(inputs=net,
pool_size=[13, 13],
strides=1,
data_format=data_format)
# TODO fix for data_format later
logits = tf.squeeze(net, [2, 3])
return logits
def dense_layer(x,
n_out,
activation=ACTIVATION,
drop_rate=0.0,
reuse=None,
name=None,
batch_norm=False,
regularize=False,
training=True):
wt_init = None
bi_init = None
#wt_init = tf.truncated_normal_initializer(stddev=0.15)
#bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.25)
#wt_init = tf.truncated_normal_initializer(stddev=0.05)
#bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.35)
if regularize:
wt_reg = WT_REG
bi_reg = BI_REG
else:
wt_reg = None
bi_reg = None
# Apply dense layer
y = layers.dense(x,
n_out,
activation=None,
use_bias=True,
kernel_initializer=wt_init,
bias_initializer=bi_init,
kernel_regularizer=wt_reg,
bias_regularizer=bi_reg,
trainable=True,
name=name,
reuse=reuse)
# Apply batch normalization
if batch_norm:
if name:
y = batch_norm_layer(y, training, name=name + '_bn', reuse=reuse)
else:
y = batch_norm_layer(y, training, name=name, reuse=reuse)
# Apply dropout
y = layers.dropout(y, rate=drop_rate, training=training)
# Apply activation
if activation is not None:
y = activation(y)
return y
def misconception_fishing(inputs,
filters_list,
kernel_size,
strides_list,
objective_function,
training,
pre_filters=128,
post_filters=128,
post_layers=1,
dropout_rate=0.5,
internal_dropout_rate=0.5,
other_objectives=(),
feature_means=None,
feature_stds=None):
_, layers = misconception_model(inputs,
filters_list,
kernel_size,
strides_list,
training,
other_objectives,
sub_filters=post_filters,
sub_layers=2,
dropout_rate=internal_dropout_rate,
feature_means=feature_means,
feature_stds=feature_stds)
expanded_layers = []
for i, lyr in enumerate(layers):
lyr = ly.conv1d(lyr, pre_filters, 1, activation=None)
lyr = ly.batch_normalization(lyr, training=training)
lyr = tf.nn.relu(lyr)
expanded_layers.append(repeat_tensor(lyr, 2**i))
embedding = tf.add_n(expanded_layers)
for _ in range(post_layers - 1):
embedding = ly.conv1d(embedding,
post_filters,
1,
activation=None,
use_bias=False)
embedding = ly.batch_normalization(embedding, training=training)
embedding = tf.nn.relu(embedding)
embedding = ly.conv1d(embedding, post_filters, 1, activation=tf.nn.relu)
embedding = ly.dropout(embedding, training=training, rate=dropout_rate)
fishing_outputs = ly.conv1d(embedding, 1, 1, activation=None)
return objective_function.build(fishing_outputs)
def conv1d_layer(x,
n_out,
kernel_size,
stride=1,
activation=ACTIVATION,
regularize=True,
use_bias=True,
drop_rate=0.0,
batch_norm=BATCH_NORM,
training=True,
name=None,
reuse=None):
if batch_norm:
if name:
x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse)
else:
x = batch_norm_layer(x, training, name=name, reuse=reuse)
#wt_init = tf.truncated_normal_initializer(stddev=0.2)
#bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01)
wt_init = None
bi_init = None
if regularize:
wt_reg = WT_REG
bi_reg = BI_REG
else:
wt_reg = None
bi_reg = None
y = layers.conv1d(x,
n_out,
kernel_size,
strides=stride,
padding='same',
data_format='channels_last',
dilation_rate=1,
activation=activation,
use_bias=use_bias,
kernel_initializer=wt_init,
bias_initializer=bi_init,
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name=None,
reuse=None)
y = layers.dropout(y, rate=drop_rate, training=training)
return y
def build(self, input):
self.conv1 = self.conv_block(input, 64, "conv1")
pool1 = max_pooling2d(self.conv1, pool_size=(2, 2), strides=2)
drop1 = dropout(pool1, .25)
self.conv2 = self.conv_block(drop1, 128, 'conv2')
pool2 = max_pooling2d(self.conv2, pool_size=(2, 2), strides=2)
drop2 = dropout(pool2, .25)
self.conv3 = self.conv_block(drop2, 256, 'conv3')
pool3 = max_pooling2d(self.conv3, pool_size=(2, 2), strides=2)
drop3 = dropout(pool3, .25)
self.conv4 = self.conv_block(drop3, 512, 'conv4')
pool4 = max_pooling2d(self.conv4, pool_size=(2, 2), strides=2)
drop4 = dropout(pool4, .25)
self.conv5 = conv2d(drop4,
1024,
kernel_size=(3, 3),
activation=tf.nn.relu,
padding='SAME')
self.conv5_2 = conv2d(self.conv5,
1024,
kernel_size=(3, 3),
activation=tf.nn.relu,
padding='SAME')
self.deconv4 = self.deconv_block(self.conv5_2, 512, self.conv4, "SAME",
'deconv4')
self.deconv3 = self.deconv_block(self.deconv4, 256, self.conv3,
"VALID", 'deconv3')
self.deconv2 = self.deconv_block(self.deconv3, 128, self.conv2, "SAME",
'deconv2')
self.deconv1 = self.deconv_block(self.deconv2, 64, self.conv1, "VALID",
'deconv1')
self.output = conv2d(self.deconv1,
filters=1,
kernel_size=1,
name='logits')
def block8(self, x):
initial = x
with tf.variable_scope("block8"):
with tf.variable_scope("branch0"):
x0 = self.basic_conv2d(x,
192,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
with tf.variable_scope("branch1"):
x1 = self.basic_conv2d(x,
192,
kernel_size=1,
stride=1,
padding="same",
namescope="conv1",
use_bias=False)
x1 = self.basic_conv2d(x1,
224,
kernel_size=(1, 3),
stride=1,
padding="same",
namescope="conv2",
use_bias=False)
x1 = self.basic_conv2d(x1,
256,
kernel_size=(3, 1),
stride=1,
padding="same",
namescope="conv3",
use_bias=False)
x = tf.concat([x0, x1], axis=-1)
x = layers.conv2d(x,
filters=2080,
kernel_size=1,
padding="same",
strides=1,
use_bias=True)
x = initial + x
x = layers.dropout(x, noise_shape=[None, 1, 1, None])
return x
def vgg(inputs, ys, knn_inputs, knn_ys):
# block 1
x = conv2d(inputs, filters=64, kernel_size=(3, 3), padding='same', activation=ReLU)
x = conv2d(x, filters=64, kernel_size=(3, 3), padding='same', activation=ReLU)
x = max_pooling2d(x, pool_size=(2, 2), strides=(2, 2))
# block 2
x = conv2d(x, filters=128, kernel_size=(3, 3), padding='same', activation=ReLU)
x = conv2d(x, filters=128, kernel_size=(3, 3), padding='same', activation=ReLU)
x = max_pooling2d(x, pool_size=(2, 2), strides=(2, 2))
# final
x = flatten(x)
x = dense(x, units=1024, activation=ReLU)
x = dropout(x, rate=0.5, training=)
x = dense(x, units=1024, activation=ReLU)
x = dense(x, units=self.data_loader.num_outputs)
return x
def dense1d_layer(x,
n_out,
activation=ACTIVATION,
regularize=True,
use_bias=True,
drop_rate=0.0,
batch_norm=BATCH_NORM,
training=True,
name=None,
reuse=None):
if batch_norm:
if name:
x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse)
else:
x = batch_norm_layer(x, training, name=name, reuse=reuse)
#wt_init = tf.truncated_normal_initializer(stddev=0.2)
#bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01)
wt_init = None
bi_init = None
if regularize:
wt_reg = WT_REG
bi_reg = BI_REG
else:
wt_reg = None
bi_reg = None
y = layers.dense(x,
n_out,
activation=activation,
use_bias=True,
kernel_initializer=wt_init,
bias_initializer=bi_init,
kernel_regularizer=wt_reg,
bias_regularizer=bi_reg,
trainable=True,
name=name,
reuse=reuse)
y = layers.dropout(y, rate=drop_rate, training=training)
return y
def lstm_model(features, mode):
"""
cnn model structure
:param features: images
:return: predicts
"""
input_layer = tf.unstack(value=features, num=28, axis=1, name='input')
lstm_cell = rnn.BasicLSTMCell(num_units=128, name='lstm')
lstm_out, _ = rnn.static_rnn(
lstm_cell,
input_layer,
dtype=tf.float32,
)
flatten_layer = layers.flatten(lstm_out[-1], name='flatten')
dense_layer = layers.dense(inputs=flatten_layer, units=512, name='dense')
dropout = layers.dropout(inputs=dense_layer,
rate=0.5,
training=(mode == tf.estimator.ModeKeys.TRAIN),
name='dropout')
logits = layers.dense(inputs=dropout, units=10, name='logits')
return logits
def cnn_model(features, mode):
"""
cnn model structure
:param features: images
:return: predicts
"""
input_layer = tf.reshape(features, shape=[-1, 28, 28, 1], name='input')
# conv1
# trainable can change in middle of training
conv1 = layers.conv2d(inputs=input_layer, filters=32, kernel_size=[3, 3], padding="same", activation=tf.nn.relu,
name='conv1')
pool1 = layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2, name='pool1')
# conv2
conv2 = layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5], padding="same", activation=tf.nn.relu,
name='conv2')
pool2 = layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2, name='pool2')
# fully connected layer
pool2_flat = layers.flatten(pool2, name='flatten')
dense = layers.dense(inputs=pool2_flat, units=512, activation=tf.nn.relu, name='dense_layer')
dropout = layers.dropout(inputs=dense, rate=0.4, training=(mode == tf.estimator.ModeKeys.TRAIN), name='dropout')
# output layer
logits = tf.layers.dense(inputs=dropout, units=10, name='logits')
return logits
def inference(self, input):
output = conv2d(input,
self.output_channels,
kernel_size=(3, 1),
strides=1,
padding="SAME")
output = conv2d(output,
self.output_channels,
kernel_size=(1, 3),
strides=1,
padding="SAME",
activation=None)
output = batch_normalization(output, self.training)
output = tf.nn.relu(output)
output = conv2d(output,
self.output_channels,
kernel_size=(3, 1),
strides=1,
dilation_rate=(self.dilated, self.dilated),
padding="SAME")
output = conv2d(output,
self.output_channels,
kernel_size=(1, 3),
strides=1,
dilation_rate=(self.dilated, self.dilated),
padding="SAME",
activation=None)
output = batch_normalization(output, self.training)
if self.dropprob != 0:
output = dropout(output,
rate=self.dropprob,
training=self.training)
return tf.nn.relu(output + input)
def googlenet(X, config):
""" googlenet implementation.
"""
with tf.name_scope('googlenet'):
block1 = conv2d(X, 64, 7, strides=2, padding='SAME')
block1 = tf.nn.relu(block1)
block1 = max_pooling2d(block1, 3, 2, padding='SAME')
block1 = tf.nn.local_response_normalization(block1)
block2 = conv2d(block1, 64, 1, padding='SAME')
block2 = tf.nn.relu(block2)
block2 = conv2d(block2, 192, 3, padding='SAME')
block2 = tf.nn.relu(block2)
block2 = tf.nn.local_response_normalization(block2)
block2 = max_pooling2d(block2, 3, 2, padding='SAME')
block2 = tf.nn.relu(block2)
# inception x2
block3 = inceptionBlock(block2,
c1=64,
c3_r=96,
c3=128,
c5_r=16,
c5=32,
p3_r=32)
block3 = inceptionBlock(block3,
c1=128,
c3_r=128,
c3=192,
c5_r=32,
c5=96,
p3_r=64)
block3 = max_pooling2d(block3, 3, 2, padding='SAME')
# inception x5
block4 = inceptionBlock(block3,
c1=192,
c3_r=96,
c3=208,
c5_r=16,
c5=48,
p3_r=64)
block4 = inceptionBlock(block4,
c1=160,
c3_r=112,
c3=224,
c5_r=24,
c5=64,
p3_r=64)
block4 = inceptionBlock(block4,
c1=128,
c3_r=128,
c3=256,
c5_r=24,
c5=64,
p3_r=64)
block4 = inceptionBlock(block4,
c1=112,
c3_r=144,
c3=288,
c5_r=32,
c5=64,
p3_r=64)
block4 = inceptionBlock(block4,
c1=256,
c3_r=160,
c3=320,
c5_r=32,
c5=128,
p3_r=128)
block4 = max_pooling2d(block4, 3, 2, padding='SAME')
# inception x2 with average pooling
block5 = inceptionBlock(block4,
c1=256,
c3_r=160,
c3=320,
c5_r=32,
c5=128,
p3_r=128)
block5 = inceptionBlock(block5,
c1=384,
c3_r=192,
c3=384,
c5_r=48,
c5=128,
p3_r=128)
block5 = average_pooling2d(block5, 7, 1, padding='SAME')
block5 = dropout(block5, rate=0.4, training=config.training)
logits = flatten(block5)
logits = dense(logits, config.NUM_CLASSES)
return logits, None
def cnn_model(features, labels, mode):
""" CNN MODEL"""
""" ((Conv2D)*2-->MaxPool-->Dropout)*2-->Dense-->dropout-->softmax """
# input layer
input_layer = tf.reshape(features["x"], (-1, 28, 28, 1))
# conv1
conv1 = layers.conv2d(inputs=input_layer,
filters=32,
kernel_size=(5, 5),
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# conv2
conv2 = layers.conv2d(inputs=conv1,
filters=32,
kernel_size=(5, 5),
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# max1
max1 = layers.max_pooling2d(inputs=conv2, pool_size=(2, 2), strides=2)
# drop1
drop1 = layers.dropout(inputs=max1,
rate=0.25,
training=(mode == tf.estimator.ModeKeys.TRAIN))
# conv3
conv3 = layers.conv2d(inputs=drop1,
filters=64,
kernel_size=(3, 3),
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# conv4
conv4 = layers.conv2d(inputs=conv3,
filters=64,
kernel_size=(3, 3),
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# max1
max2 = layers.max_pooling2d(inputs=conv4, pool_size=(2, 2), strides=2)
# drop1
drop2 = layers.dropout(inputs=max2,
rate=0.25,
training=(mode == tf.estimator.ModeKeys.TRAIN))
# Dense layer
flat1 = tf.reshape(drop2, (-1, 7 * 7 * 64))
dense1 = layers.dense(inputs=flat1, units=256, activation=tf.nn.relu)
drop3 = layers.dropout(inputs=dense1, rate=0.5)
logits = layers.dense(inputs=drop3, units=10)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits=logits, name="softmax_tensor"),
} # type: Dict[str, Any]
# if mode is eval of predict return this dict
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate the loss
loss = tf.losses.softmax_cross_entropy(onehot_labels=tf.one_hot(
indices=tf.cast(labels, tf.int32), depth=10),
logits=logits)
# Optimize for training process
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
eval_metric_ops = {
"accuracy":
tf.metrics.accuracy(labels=labels, predictions=predictions["classes"])
}
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=eval_metric_ops)
def __call__(self, inputs):
return layers.dropout(inputs, training=self._is_training)
def VGG_net(x, is_training, params):
with tf.variable_scope('vgg_16', 'vgg_16', [x]) as sc:
layer1 = ConvBatch(x, 64, is_training, "layer1")
layer2 = ConvBatch(layer1, 64, is_training, "layer2")
maxpool_2 = layers.max_pooling2d(layer2, pool_size=2, strides=2)
dropout_2 = layers.dropout(maxpool_2,
rate=params['dropout'],
training=is_training)
layer3 = ConvBatch(dropout_2, 128, is_training, "layer3")
layer4 = ConvBatch(layer3, 128, is_training, "layer4")
maxpool_4 = layers.max_pooling2d(layer4, pool_size=2, strides=2)
dropout_4 = layers.dropout(maxpool_4,
rate=params['dropout'],
training=is_training)
layer5 = ConvBatch(dropout_4, 256, is_training, "layer5")
layer6 = ConvBatch(layer5, 256, is_training, "layer6")
layer7 = ConvBatch(layer6, 256, is_training, "layer7")
maxpool_7 = layers.max_pooling2d(layer7, pool_size=2, strides=2)
dropout_7 = layers.dropout(maxpool_7,
rate=params['dropout'],
training=is_training)
layer8 = ConvBatch(dropout_7, 512, is_training, "layer8")
layer9 = ConvBatch(layer8, 512, is_training, "layer9")
layer10 = ConvBatch(layer9, 512, is_training, "layer10")
maxpool_10 = layers.max_pooling2d(layer10, pool_size=2, strides=2)
dropout_10 = layers.dropout(maxpool_10,
rate=params['dropout'],
training=is_training)
layer11 = ConvBatch(dropout_10, 512, is_training, "layer11")
layer12 = ConvBatch(layer11, 512, is_training, "layer12")
layer13 = ConvBatch(layer12, 512, is_training, "layer13")
maxpool_13 = layers.max_pooling2d(layer13, pool_size=2, strides=2)
dropout_13 = layers.dropout(maxpool_13,
rate=params['dropout'],
training=is_training)
flat_output = layers.flatten(dropout_13)
dense14 = DenseBatch(flat_output, 4096, is_training, "dense14")
dropout_14 = layers.dropout(dense14,
rate=params['dropout'],
training=is_training)
dense15 = DenseBatch(dropout_14, 4096, is_training, "dense15")
logits = layers.dense(
dense15,
100,
activation=None,
kernel_initializer=tf.contrib.layers.xavier_initializer())
return logits
def main():
# parsing
parser = argparse.ArgumentParser()
parser.add_argument(
"input", help="pretrained sqznet weight file, e.g., sqz_full.mat")
parser.add_argument("output", help="folder for generated model")
parser.add_argument(
"train", help="train folder dir with images in different subfolders")
parser.add_argument("--test", help="optional test folder")
args = parser.parse_args()
# data loading
label2class = {"green": 0, "yellow": 1, "red": 2, "nolight": 3}
train_dir = args.train
test_dir = args.test or args.train
train_imgs, train_targets = load_imgs_labels(train_dir, label2class)
print("loaded training images", train_imgs.shape)
if test_dir != train_dir:
test_imgs, test_targets = load_imgs_labels(test_dir, label2class)
else:
test_imgs, test_targets = train_imgs, train_targets
print("loaded test images", test_imgs.shape)
## model
# load weights as numpy array
sqz_wts, sqz_mean = load_net(args.input)
params = {"dropout_rate": 0.5, "n_classes": 4, "learning_rate": 5e-4}
tf.reset_default_graph()
image = tf.placeholder(tf.float32, shape=[None, 227, 227, 3])
labels = tf.placeholder(tf.int32, shape=[None])
training = tf.placeholder(tf.bool, shape=[])
sqz_net = net_preloaded(sqz_wts, image, 'max',
False) # weights as tf.constant
bottleneck_feats = sqz_net['fire9/concat_conc'] # ?x13x13x512
output = layers.dropout(bottleneck_feats,
rate=params["dropout_rate"],
training=training)
output = layers.conv2d(output,
params["n_classes"],
kernel_size=[13, 13],
strides=[1, 1])
logits = tf.reshape(output, [-1, 4])
predictions = tf.argmax(logits, axis=1)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(labels, logits))
match = tf.nn.in_top_k(logits, labels, 1)
accuracy = tf.reduce_mean(tf.cast(match, tf.float32))
optimizer = tf.train.AdamOptimizer(learning_rate=params["learning_rate"])
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
saver = tf.train.Saver()
tf.add_to_collection("model_input", image)
tf.add_to_collection("model_input", labels)
tf.add_to_collection("model_input", training)
tf.add_to_collection("model_output", logits)
tf.add_to_collection("model_output", predictions)
## training
batch_size = 256
n_epochs = train_imgs.shape[0] // batch_size * 10
train_batches = make_tl_generator(train_imgs,
train_targets,
batch_size=batch_size)
test_batches = make_tl_generator(test_imgs,
test_targets,
batch_size=batch_size)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(n_epochs):
x_batch, y_batch = next(train_batches)
_, loss_val = sess.run([train_op, loss],
feed_dict={
image: x_batch,
labels: y_batch,
training: True
})
if epoch % 50 == 0:
print(epoch, loss_val)
model_file = saver.save(sess, path.join(args.output, "tl_model"))
print("model saved to", model_file)
## test accuracy
test_preds, test_accuracy = [], []
for i in range(0, test_imgs.shape[0], batch_size):
tp, ta = sess.run(
[predictions, accuracy],
feed_dict={
image: test_imgs[i:i + batch_size],
labels: test_targets[i:i + batch_size],
training: False
})
test_preds.append(tp)
test_accuracy.append(ta)
test_yhat = np.concatenate(test_preds)
test_accuracy = np.mean(test_yhat == test_targets)
print("test accuracy:", test_accuracy)
