def forward(self, inputs, scope=None):
with tf.Variable_scope(scope, 'StaticBrnn', [inputs]) as scope:
# inputs [B, max_time, depth]
inputs_list = tf.unstack(inputs, axis=1) # [B, depth]
outputs_list, _, _ = tf.nn.static_bidirectional_rnn(
self._fw_cell, self._bw_cell, inputs_list, dtype=tf.float32)
# 正则化
filter_weights = lambda vars: [
x for x in vars if x.op.name.endswith('kernel')
]
tf.contrib.layers.apply_regularization(
self._rnn_regularizer,
filter_weights(self._fw_cell.trainable_weights))
tf.contrib.layers.apply_regularization(
self._rnn_regularizer,
filter_weights(self._bw_cell.trainable_weights))
rnn_outputs = tf.stack(outputs_list,
axis=1) # [B, max_time, depth]
# 全连接层
if self._num_output_units > 0:
with arg_scope(self._fc_hyperparams):
rnn_outputs = fully_connected(rnn_outputs,
self._num_output_units,
activation_fn=tf.nn.relu)
if self._summarize_activations:
for i in range(len(outputs_list)):
tf.summary.histogram(
'Activation/{}/Step_{}'.format(scope.name, i),
outputs_list[i])
return rnn_outputs
def neural_network(model = 'lstm', rnn_size=128, num_layers=2):
if model == 'rnn':
cell_fun = tf.nn.rnn_cell.BasicRNNCell
elif model == 'gru':
cell_fun = tf.nn.rnn_cell.GRUCell
elif model == 'lstm':
cell_fun = tf.nn.rnn_cell.BasicLSTMCell
cell = cell_fun(run_size, state_is_tuple=True)
cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers, state_is_tuple=True)
initial_state = cell.zero_state(batch_size, tf.float32)
with tf.Variable_scope('rnnlm'):
softmax_w = tf.get_variable('softmax_w', [rnn_size, len(words)+1])
softmax_b = tf.get_variable('softmax_b', [len(words)+1])
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [len(words)+1, rnn_size])
inputs = tf.nn.embedding_lookup(embedding, input_data)
outputs, last_state = tf.nn.dynamic_rnn(cell,inputs,initial_state=initial_state,scope='rnnlm')
output = tf.reshape(outputs, [-1, rnn_size])
logits = tf.matmul(output, softmax_w) + softmax_b
probs = tf.nn.softmax(logits)
return logits, last_state, probs, cell, initial_state
def discriminator(inputs, labels, reuse=None):
with tf.Variable_scope('discriminator') as scope:
if reuse:
scope.reuse_variable()
inputs = tf.concat([inputs, labels], 1)
hidden = tf.layers.dense(inputs, n_hidden, activation =tf.nn.relu)
output = tf.layers.dense(hidden, 1, activation=None)
return output
def cross_layer2(x0, x, name):
with tf.Variable_scope(name):
input_dim = x0.get_shape().as_list[1]
w = tf.get_variable(
"weight", [input_dim],
initializer=tf.truncated_normal_initializer(stddev=0.01))
b = tf.get_variable(
"bias", [input_dim],
initializer=tf.truncated_normal_initializer(stddev=0.01))
xb = tf.tensordot(tf.reshape(x, [-1, 1, input_dim]), w, 1)
return x0 * xb + b + x
def cross_layer(x0, x, name):
with tf.Variable_scope(name):
input_dim = x0.get_shape().as_list[1]
w = tf.get_variable(
"weight", [input_dim],
initializer=tf.truncated_normal_initializer(stddev=0.01))
b = tf.get_variable(
"bias", [input_dim],
initializer=tf.truncated_normal_initializer(stddev=0.01))
xx0 = tf.expand_dims(x0, -1)
mat = tf.matmul(xx0, tf.reshape(x, [-1, 1, input_dim]))
return tf.tensordot(mat, w, 1) + b + x
def fc_layer(self, x, name):
with tf.Variable_scope(name):
shape = x.get_shape().as_list()
dim = 1
for i in shape[1:]:
dim *= 1
x = tf.reshape(x, [-1, dim])
w = self.get_fc_weight(name)
b = self.get_bias(name)
result = tf.nn.bias_add(tf.matmul(x, w), b)
return result
def _build_net(self):
# ------ build evaluate_net -----
self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s')
self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculate loss
with tf.variable_scope('eval_net'):
# c_names(collections_names) 在更新target_net参数时会用到
# n_l1是第一层神经元的数量，w_initializer, b_initializer分别是weight和bias的初始化值
# TODO: 为什么w和b的初始化值只有单个值？不应该是矩阵和向量吗？
c_names, n_l1, w_initializer, b_initializer = ['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 10, tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1)
# eval_net第一层，collections在更新target_net参数时用到
with tf.variable_scope('l1'):
w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names) # collections属性是一个graph collections keys的列表，用于添加变量至对应名字下的collection中
b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
l1 = tf.nn.relt(tf.matmul(self.s, w1) + b1)
# eval_net第二层，collections在更新target_net参数时会用到
with tf.variable_scope('l2'):
w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
self.q_eval = tf.matmul(l1, w2) + b2
with tf.Variable_scope('loss'):
self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
with tf.variable_scope('train'):
self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
# ----- build target_net -----
self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_')
with tf.variable_scope('target_net'):
# c_names(collections_names) are the collections to store variables
c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
# 第一层,c_names(collections_names)在更新target_net参数时会用到
with tf.variable_scope('l1'):
w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
l1 = tf.nn.relu(tf.matmul(self.s_, w1) + b1)
# 第一层,c_names(collections_names)在更新target_net参数时会用到
with tf.variable_scope('l2'):
w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
b2 = tf.get_variable('b2', [1, self.n_actions], initializer=w_initializer, collections=c_names)
self.q_next = tf.matmul(l1, w2) + b2
def add_projections(self, node_tensors):
"""Add projections to the composition vectors to compute the raw sentiment scores
Hint: Reuse the "Projection" variable_scope here
Args:
node_tensors: tensor(?, embed_size)
Returns:
output: tensor(?, label_size)
"""
logits = None
### YOUR CODE HERE
with tf.Variable_scope("Projection", reuse=True):
U = tf.get_variable("U")
bs = tf.get_variable("bs")
logits = tf.matmul(node_tensors, U) + bs
### END YOUR CODE
return logits
def conv_net(x_dict,n_classes,dropout,reuse,is_training):
with tf.Variable_scope('ConvNet',reuse=reuse):
x = x_dict['images']
x = tf.reshape(x,shape=[-1,28,28,1])
conv1 = tf.layers.conv2d(x,32,5,activation=tf.nn.relu)
conv1 = tf.layers.max_pooling2d(conv1,2,2)
conv2 = tf.layers.conv2d(conv1,64,3,activation=tf.nn.relu)
conv2 = tf.layers.max_pooling2d(conv2,2,2)
fc1 = tf.contrib.layers.flatten(conv2)
fc1 = tf.layers.dense(fc1,1024)
fc1 = tf.layers.dropout(fc1,rate=dropout,training=is_training)
out = tf.layers.dense(fc1,n_classes)
return out
def inception_v3_base(inputs,scope=None):
# key point saved in end_points
end_points ={}
with tf.Variable_scope(scope,'InceptionV3',[inputs]):
# using arg_scope set the parameer for the conv2d,max_pool2d,avg_pool2d
with slim.arg_scope([slim.conv2d,slim.max_pool2d,slim.avg_pool2d],stride=1,padding="VALID"):
# the part not in inception module
# construction (tensor,output passage,convolution kernel number,stride ,padding)
# if input 299*299*3 output is 35*35*192,Spatial dimension is less and output passway is more
net = slim.conv2d(inputs,32,[3,3],stride=2,scope='Conv2d_1a_3x3')
net = slim.conv2d(net,32,[3,3],scope='Conv2d_2a_3x3')
net = slim.conv2d(net,64,[3,3],padding="SAME",scope='Conv2d_2b_3x3')
net = slim.max_pool2d(net,[3,3],stride=2,scope="Maxpool_3a_3x3")
net = slim.conv2d(net,80,[1,1],scope='Conv2d_3b_1x1')
net = slim.conv2d(net,192,[3,3],scope="Conv2d_4a_3x3")
net = slim.max_pool2d(net,[3,3],stride=2,scope="MaxPool_5a_3x3")
with slim.arg_scope([slim.conv2d,slim.max_pool2d,slim.avg_pool2d],stride=1,padding='SAME'):
with tf.variable_scope('Mixed_5b'):
# inception module mixed_5b have branch0--branch3
with tf.variable_scope('Branch_0'):
branch_0 =slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,48,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,64,[5,5],scope='Conv2d_0b_5x5')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0b_3x3')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0c_3x3')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,32,[1,1],scope='Conv2d_0b_1x1')
# concat can merge passway so the number of passway have 32+96+48+64
# tensor also 35*35
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_5c'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,48,[1,1],scope='Conv2d_0b_1x1')
branch_1 = slim.conv2d(branch_1,64,[5,5],scope='Conv_1_0c_5x5')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0b_3x3')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0c_3x3')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,64,[1,1],scope="Conv2d_0b_1x1")
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_5d'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,48,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,64,[5,5],scope='Conv2d_0b_5x5')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0b_3x3')
branch_2 = slim.conv2d(branch_2,96,[3,3],scope='Conv2d_0c_3x3')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,64,[1,1],scope="Conv2d_0b_1x1")
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_6a'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,384,[1,1],stride=2,padding='VALID',scope='Conv2d_1a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,64,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,96,[3,3],scope='Conv2d_0b_5x5')
branch_1 = slim.conv2d(branch_1,96,[3,3],stride=2,padding='VALID',scope="Conv2d_1a_1x1")
with tf.variable_scope('Branch_2'):
branch_2 = slim.max_pool2d(net,[3,3],stride=2,padding='VALID',scope='MaxPool_1a_1x1')
net = tf.concat([branch_0,branch_1,branch_2],3)
with tf.variable_scope('Mixed_6b'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,128,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,128,[1,7],scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,192,[7,1],scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,128,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,128,[7,1],scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,128,[1,7],scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,128,[7,1],scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,192,[1,7],scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope="AvgPool_0a_3x3")
branch_3 = slim.conv2d(branch_3,192,[1,1],scope='Conv2d_0b_1x1')
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_6c'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,160,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,160,[1,7],scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,192,[7,1],scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,160,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,160,[7,1],scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,160,[1,7],scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,160,[7,1],scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,192,[1,7],scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope="AvgPool_0a_3x3")
branch_3 = slim.conv2d(branch_3,192,[1,1],scope='Conv2d_0b_1x1')
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_6d'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,160,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,160,[1,7],scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,192,[7,1],scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,128,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,160,[7,1],scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,160,[1,7],scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,160,[7,1],scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,192,[1,7],scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope="AvgPool_0a_3x3")
branch_3 = slim.conv2d(branch_3,192,[1,1],scope='Conv2d_0b_1x1')
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
with tf.variable_scope('Mixed_6e'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,192,[1,7],scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,192,[7,1],scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,192,[7,1],scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,192,[1,7],scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,192,[7,1],scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,192,[1,7],scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net,[3,3],scope="AvgPool_0a_3x3")
branch_3 = slim.conv2d(branch_3,192,[1,1],scope='Conv2d_0b_1x1')
net = tf.concat([branch_0,branch_1,branch_2,branch_3],3)
end_points['Mixed_6e'] = net
with tf.variable_scope('Mixed_7a'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
branch_0 = slim.conv2d(branch_0,320,[3,3],stride=2,padding='VALID',scope='Conv2d_1a_3x3')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,192,[1,1],scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(banch_1,192,[1,7],scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,192,[7,1],scope='Conv2d_0c_7x1')
branch_1 = slim.conv2d(branch_1,192,[3,3],stride=2,padding='VALID',scope='Conv2d_1a_3x3')
with tf.variable)scope('Branch_2'):
branch_2 = slim.max_pool2d(net,[3.3],stride=2,padding='VALID',scope='MaxPool_1a_3x3')
net = tf.concat([branch_0,branch_1,branch_2],3)
with tf.variable_scope('Mixed_7b'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,320,[1,1],scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,384,[1,1],scope='Conv2d_0a_1x1')
branch_1 = tf.concat([
slim.conv2d(branch_1,384,[1,3],scope="Conv2d_0b_1x3"),
slim.conv2d(branch_1,384,[3,1],scope='Conv2d_0b_3x1')],3)
with tf.variable)scope('Branch_2'):
branch_2 = slim.conv2d(net,448,[1.1],scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,384,[3,3],scope='Conv2d_0b_3x3')
branch_2 = tf.concat([
slim.conv2d(branch_2,384,[1,3],scope='Conv2d_0c_1x3'),
slim.conv2d(branch_2,384,[3,1],scope="Conv2d_0d_3x1")],3)
def mobilenet(inputs,
num_classes=1000,
is_training=True,
width_multiplier=1,
scope='MobileNet'):
'''
Args:
inputs: a tensor of size [batch_size,height,width,channels]
num_classes: number of predicted classes
is_training: model is being trained
scope: scope for the variables
Returns:
logits
end_points
'''
def _depthwise_separable_conv(inputs,
num_pwc_filters,
width_multiplier,
sc,
downsample=False):
num_pwc_filters = round(num_pwc_filters * width_multiplier)
_stride = 2 if downsample else 1
#skip pointwise by setting num_output=None
depthwise_conv = slim.separable_convolution2d(inputs,
num_outputs=None,
stride=_stride,
depth_multiplier=1,
kernel_size=[3, 3],
scope=sc +
'/depthwise_conv')
bn = slim.batch_norm(depthwise_conv, scope=sc + '/dw_batch_norm')
pointwise_conv = slim.convolution2d(bn,
num_pwc_filters,
kernel_size=[1, 1],
scope=sc + '/pointwise_conv')
bn = slim.batch_norm(pointwise_conv, scope=sc + '/pw_batch_norm')
return bn
with tf.Variable_scope(scope) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([slim.convolution2d, slim.separable_convolution2d],
activation_fn=None,
outputs_collections=[end_points_collection]):
with slim.arg_scope([slim.batch_norm],
is_training=is_training,
activation_fn=tf.nn.relu,
fused=True):
net = slim.convolution2d(inputs,
round(32 * width_multiplier), [3, 3],
stride=2,
padding='same',
scope='conv_1')
net = slim.batch_norm(net, scope='conv_1/batch_norm')
net = _depthwise_separable_conv(net,
64,
width_multiplier,
sc='conv_ds_2')
net = _depthwise_separable_conv(net,
128,
width_multiplier,
downsample=True,
sc='conv_ds_3')
net = _depthwise_separable_conv(net,
128,
width_multiplier,
sc='conv_ds_4')
net = _depthwise_separable_conv(net,
256,
width_multiplier,
downsample=True,
sc='conv_ds_5')
net = _depthwise_separable_conv(net,
256,
width_multiplier,
sc='conv_ds_6')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
downsample=True,
sc='conv_ds_7')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
sc='conv_ds_8')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
sc='conv_ds_9')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
sc='conv_ds_10')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
sc='conv_ds_11')
net = _depthwise_separable_conv(net,
512,
width_multiplier,
sc='conv_ds_12')
net = _depthwise_separable_conv(net,
1024,
width_multiplier,
downsample=True,
sc='conv_ds_13')
net = _depthwise_separable_conv(net,
1024,
width_multiplier,
sc='conv_ds_14')
net = slim.avg_pool2d(net, [7, 7], scope='avg_pool_15')
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
logits = slim.fully_connected(net,
num_classes,
activation_fn=None,
scope='fc_16')
predictions = slim.softmax(logits, scope='predictions')
end_points['Logits'] = logits
end_points['Predictions'] = predictions
return logits, end_points
data_y = np.zeros([batch_size, batch_pation_length], dtype= np.int32)# 200 * 50000
for i in range(batch_size):
data_x = raw_x[batch_pation_length*i: batch_pation_length*(i+1)]
data_y = raw_y[batch_pation_length*i: batch_pation_length*(i+1)]
epoch_size = batch_pation_length // num_step #迭代次数
for i in range(epoch_size):
x = data_x[:, i*num_step:(i+1)*num_step]
y = data_y[:, i*num_step:(i+1)*num_step]
yield (x,y) #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
def gen_epochs(n, num_steps):
for i in range(n):
yield gen_batch(gen_data(),batch_size, num_step=num_steps)
#定义输入输出格式
x = tf.placeholder(tf.int32, [batch_size, numstep], name='x') # 200*5
y = tf.placeholder(tf.int32, [batch_size, numstep], name='y') # 200*5
init_state = tf.zeros([batch_size, state_size]) #初始化状态， 200*4
# rnn输入
x_one_hot = tf.one_hot(x, num_class)
rnn_input = tf.unstack(x_one_hot, axis=1)
#RNN输入
with tf.Variable_scope('rnn_cell'):
W = tf.get_variable('W', [num_class + state_size, state_size])
b = tf.get_variable('b', [state_size], initializer=tf.constant_initializer(0.0))
x = tf.placeholder(shape=[batch_size, word_size], dtype=tf.int32)
y_ = tf.placeholder(shape=[batch_size], dtype=tf.float32)
embeddings = tf.constant(embeddings)
embed = tf.nn.embedding_lookup(embeddings, x)
gru = tf.nn.rnn_cell.GRUCell(num_units=16,
reuse=tf.AUTO_REUSE,
activation=tf.nn.elu)
state = gru.zero_state(batch_size, dtype=tf.float32)
lis = []
with tf.variable_scope('RNN'):
for timestep in range(word_size):
if timestep == 1:
tf.Variable_scope().reuse_variables()
(cell_output, state) = gru(embed[:, timestep], state)
out_put = state
lay1 = tf.nn.elu(ml.layer_basic(out_put, 4))
lay2 = ml.layer_basic(ml.bn_with_wb(lay1), 1)
y = tf.nn.sigmoid(lay2[:, 0])
loss = tf.reduce_sum(-y_ * tf.log(y + 0.000000001) - (1 - y_) *
tf.log(1 - y + 0.00000001)) / batch_size / tf.log(2.0)
optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
# ...................................................................
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('begin..................................', sum(y_test) / len(y_test))
def inception_v3_base(inputs, scope=None):
end_points = {}
with tf.variable_scope(scope, 'InceptionV3', [inputs]):
with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
stride=1,
padding='VALID'):
net = slim.conv2d(inputss,
32, [3, 3],
stride=2,
scope='Conv2d_1a_3x3')
net = slim.conv2d(net, 32, [3, 3], scope='Conv2d_2a_3x3')
net = slim.conv2d(net,
64, [3, 3],
padding='SAME',
scope='Conv2d_2b_3x3')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='MaxPool_3a_3x3')
net = slim.conv2d(net, 80, [1, 1], scope='Conv2d_3b_1x1')
net = slim.conv2d(net, 192, [3, 3], scope='Conv2d_4a_3x3')
net = slim.max_pool2d(netm[3, 3], stride=2, scope='MaxPool_5a_3x3')
####
# 4 branches 0-3
# 0. 64* 1*1 conv
# 1. 48* 1*1 then 64* 5*5 conv
# 2. 96* 3*3 conv
# 3. 3*3 mpool then 32* 1*1 conv
# tf.concat 合并输出
# 64+64+96+32=256
# output is 35*35*256
# Inception Module.1 输出的图片尺寸均为35*35，But Inception Module last 2 will change the channels
####
with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
stride=1,
padding='SAME'):
with tf.variable_scope('Mixed_5d'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
64, [1, 1],
scop='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
48, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
64, [5, 5],
scope='Conv2d_0b_5x5')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,
64, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
96, [3, 3],
scope='Conv2d 0b 3x3')
branch_2 = slim.conv2d(branch_2,
96, [3, 3],
scope='Conv2d_0c_3x3')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
32, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
#####
###
#Inception Modules 2.
###
#####
#
# 5 Inception Mudule included
#
# 1st Inception Module was named as Mixed_6a
# 3 branches
# 384* 3*3 conv
# stride 2, padding = VALID, size = 17*17
# 2ed, 3 branches 64* 1*1 and 2* 96* 3*3 conv
# 3rd, stride=2, padding=VALID, 17*17*96
#
#
# THE 3rd branches is 3*3 mpool, stride=2, padding=VALID, so the size output is 17*17*256
# use the TF.CONCAT combine the 3 branches into 1, the final output is 17*17*(384+96+256)=17*17*768
# the photo's size and channel have no changes.
#
#
with tf.variable_scope('Mixed_6a'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
384, [3, 3],
stride=2,
padding='VALID',
scope='Conv2d_1a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d('net',
64, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
96, [3, 3],
scope='Conv2d_0b_3x3')
branch_1 = slim.cnov2d(branch_1,
96, [3, 3],
stride=2,
scope='Conv2d_1a_1x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.max_pool2d(net, [3, 3],
stride=2,
padding='VALID',
scope='MaxPool_1a_3x3')
net = tf.concat([branche_0, branch_1, branch_2], 3)
#######
# Inception Modules 2.
####
# 2ed Inception --Mixed_6b
# 4 branches
# 1. 192* 1*1 conv
# 2. consist of 3 conv, 1.. 128* 1*1 2.. 128* 1*7 3.. 192* 7*1 Ｆactorization into small convolutions
# 参数量减少
# add one activation function add the non-liner
# 3. 5 conv, consist of 1*1 * 128channels , 7*1 *128, 1*7 *128, 7*1 * 128, 1*7 *192
# 4. 3*3 mpool, 1*1 *192
# tf.concat
# tensor size= 17*17*(192+192+192+192)=17*17*768
#######
with tf.variable_scope('Mixed_6b'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
128, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
128, [1, 7],
scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,
192, [7, 1],
scope='Conv2d_0c_7x1')
with tf.variable_scope('Branche_2'):
branch_2 = slim.conv2d(net,
128, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
128, [7, 1],
scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,
128, [1, 7],
scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,
128, [7, 1],
scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,
192, [1, 7],
scope='Conv2d_0e_1x7')
with tf.Variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
#####
# Inception Module -- Mixed_6c
#####
# Similar to mixed_6b
# 2ed and 3rd branch channels is different
# 128 to 160
# the final out is no change, 192
# the others is the same
# Inception module 每经过一个， the tensor size is no change, the feature is refined once, the variable conv and non-linear have great contribution to the performance.
#####
with tf.variable_scope('Mixed_6c'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
160, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
160, [1, 7],
scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,
192, [7, 1],
scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,
160, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
160, [7, 1],
scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,
160, [1, 7],
scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,
160, [7, 1],
scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,
192, [1, 7],
scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
####
# Inception Module --Mixed_6d
###
# the same to the --Mixed_6c
####
with tf.variable_scope('Mixed_6d'):
with tf.variable_scope('Branch_0'):
branch_0 = sliml.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
160, [1, 1],
scope='Conv22d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
160, [1, 7],
scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,
192, [7, 1],
scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,
160, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
160, [7, 1],
scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,
160, [1, 7],
scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,
160, [7, 1],
scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,
192, [1, 7],
scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
############
## Mix_6e
##########
## the same to the before
## Inception module 2. We put the Mixed_6e into end_points, as the Auxiliary Classifier as "Classifier"
###########
with tf.variable_scope('Mixed_6e'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
192, [1, 7],
scope='Conv2d_0b_1x7')
branch_1 = slim.conv2d(branch_1,
192, [7, 1],
scope='Conv2d_0c_7x1')
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
192, [7, 1],
scope='Conv2d_0b_7x1')
branch_2 = slim.conv2d(branch_2,
192, [1, 7],
scope='Conv2d_0c_1x7')
branch_2 = slim.conv2d(branch_2,
192, [7, 1],
scope='Conv2d_0d_7x1')
branch_2 = slim.conv2d(branch_2,
192, [1, 7],
scope='Conv2d_0e_1x7')
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
end_points['Mixed_6e'] = net
###########################################
##
## Inception Module 3.
# 3 Inception modules
# the later 2 moudles are similar
# the 1st was named as Mixed_7a, 3 branches were included
# 1..1*1 * 192 conv 3*3 * 320 conv stride=2, padding=VALID 2.. 1*1 * 192, 1*7 * 192, 7*1 * 192, 3*3 * 192, stride = 2, padding = VALID, the output is 8*8 * 192
# the 3rd branch is 3*3 mpool, stride = 2.padding=VALID output is 8*8 * 768
# tf.concat combine together The channels, 输出的tensor size 为8*8*(320+192+768)=8*8*1280
# Inception Module begins, the output size photoes is reduced , the channel is add up, tensor size is dropping.
##########################################
with tf.variable_scope('Mixed_7a'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
branch_0 = slim.conv2d(branch_0,
320, [3, 3],
stride=2,
padding='VALID',
scope='Conv2d_1a_3x3')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
192, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = slim.conv2d(branch_1,
192, [1, 7],
scope='Conv2d_ob_1x7')
branch_1 = slim.conv2d(branch_1,
192, [7, 1],
scope='Conv2d_0c_7x1')
branch_1 = slim.conv2d(branch_1,
192, [3, 3],
stride=2,
padding='VALID',
scope='Conv2d_1a_3x3')
with tf.variable_socpe('Branch_2'):
branch_2 = slim.max_pool2d(net, [3, 3],
stride=2,
padding='VALID',
scope='MaxPool_1a_3x3')
net = tf.concat([branch_0, branch_1, branch_2], 3)
########################################
##
# Inception Module 3.
# 2 Module
# 4 branches
# 1.. 1*1 *320
# 2.. 1*1 * 384 convs, and give 2 branches, 1*3 *384 and another is 3*1 * 384, then tf.concat combine the 2 branches, the outout size of tensor is 8*8 *(384+384)=8*8* 768
# 3.. 1*1 * 448 convs, then 3*3 * 384, dvivided into 2 branches: 1*3 * 384 and 3*1 * 384, and the combined as 8*8*768
# 4.. 3*3 mpool, 1*1 * 192convs,
# tf.concat the 4 branches, the tensor size is 8*8*(320+768+768+192)=8*8*2048
# the output is 2048 Channels
#######################################
with tf.variable_scope('Miexed_7b'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
320, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
384, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = tf.concat([
slim.conv2d(
branch_1, 384, [1, 3], scope='Conv2d_0b_1x3'),
slim.conv2d(
branch_1, 384, [3, 1], scope='Conv2d_0b_3x1')
], 3)
with tf.variable_scope('Branch_2'):
branch_2 = slim.conv2d(net,
448, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
384, [3, 3],
scope='Conv2d_0b_3x3')
branch_2 = tf.concat([
slim.conv2d(
branch_2, 384, [1, 3], scope='Conv2d_0c_1x3'),
slim.conv2d(
branch_2, 384, [3, 1], scope='Conv2d_0d_3x1')
], 3)
with tf.variable_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
#####################
# Mixed_7c is the last Inception Module
# But it's the copy version of the Mixed_7b, the output size is 8*8*2048
#####
# Inception Module is the result , and we return it as the inception_v3_base function's final result
#####################
with tf.varialbe_scope('Mixed_7c'):
with tf.variable_scope('Branch_0'):
branch_0 = slim.conv2d(net,
320, [1, 1],
scope='Conv2d_0a_1x1')
with tf.variable_scope('Branch_1'):
branch_1 = slim.conv2d(net,
384, [1, 1],
scope='Conv2d_0a_1x1')
branch_1 = tf.concat([
slim.conv2d(
branch_1, 384, [1, 3], scope='Conv2d_0b_1x3'),
slim.conv2d(
branch_1, 384, [3, 1], scope='Conv2d_0c_3x1')
], 3)
with tf.variale_scope('Branch_2'):
branch_2 = slim.conv2d(net,
448, [1, 1],
scope='Conv2d_0a_1x1')
branch_2 = slim.conv2d(branch_2,
384, [3, 3],
scope='Conv3d_0b_3x3')
branch_2 = tf.concat([
slim.conv2d(
branch_2, 384, [1, 3], scope='Conv2d_0c_1x3'),
slim.conv2d(
branch_2, 384, [3, 1], scope='Conv2d_0d_3x1')
], 3)
with tf.variale_scope('Branch_3'):
branch_3 = slim.avg_pool2d(net, [3, 3],
scope='AvgPool_0a_3x3')
branch_3 = slim.conv2d(branch_3,
192, [1, 1],
scope='Conv2d_0b_1x1')
net = tf.concat([branch_0, branch_1, branch_2, branch_3], 3)
return net, end_points