def inference(input_ph, istate_ph):
with tf.name_scope("inference") as scope:
weight1_var = tf.Variable(tf.truncated_normal(
[num_of_input_nodes, num_of_hidden_nodes], stddev=0.1), name="weight1")
weight2_var = tf.Variable(tf.truncated_normal(
[num_of_hidden_nodes, num_of_output_nodes], stddev=0.1), name="weight2")
bias1_var = tf.Variable(tf.truncated_normal([num_of_hidden_nodes], stddev=0.1), name="bias1")
bias2_var = tf.Variable(tf.truncated_normal([num_of_output_nodes], stddev=0.1), name="bias2")
in1 = tf.transpose(input_ph, [1, 0, 2])
in2 = tf.reshape(in1, [-1, num_of_input_nodes])
in3 = tf.matmul(in2, weight1_var) + bias1_var
in4 = tf.split(in3, length_of_sequences, 0)
cell = tf.nn.rnn_cell.BasicLSTMCell(num_of_hidden_nodes, forget_bias=forget_bias, state_is_tuple=False)
rnn_output, states_op = tf.contrib.rnn.static_rnn(cell, in4, initial_state=istate_ph)
output_op = tf.matmul(rnn_output[-1], weight2_var) + bias2_var
# Add summary ops to collect data
w1_hist = tf.summary.histogram("weights1", weight1_var)
w2_hist = tf.summary.histogram("weights2", weight2_var)
b1_hist = tf.summary.histogram("biases1", bias1_var)
b2_hist = tf.summary.histogram("biases2", bias2_var)
output_hist = tf.summary.histogram("output", output_op)
results = [weight1_var, weight2_var, bias1_var, bias2_var]
return output_op, states_op, results
def loc_net_fc(images, batch_size):
images -= 128
images /= 128.
images = tf.image.resize_images(images, 150, 150)
images_flat = tf.reshape(images, [batch_size, -1])
hidden_size = 100
with tf.name_scope('fc1') as scope:
weights = tf.Variable(tf.truncated_normal([150**2*3, hidden_size],
dtype=tf.float32, stddev=1e-3), name='weights')
biases = tf.Variable(tf.constant(0.0, shape=[hidden_size],
dtype=tf.float32), name='biases')
hidden = tf.add(tf.matmul(images_flat, weights), biases, name=scope)
hidden = tf.nn.relu(hidden)
with tf.name_scope('fc2') as scope:
weights = tf.Variable(tf.truncated_normal([hidden_size, 3],
dtype=tf.float32, stddev=1e-3), name='weights')
biases = tf.Variable(tf.constant(0.0, shape=[3], dtype=tf.float32),
name='biases')
theta = tf.add(tf.matmul(hidden, weights), biases, name=scope)
theta = tf.nn.tanh(theta)
return theta
def __init__(self,units,activation=None,mean=None,std=None,eur=False,no_bias=False):
assert not(units is None),"You need to provide the number of units ([n_in,n_out])"
if(mean is None):
mean=0.0
if(std is None):
std = 1.0/(float(units[0])**0.5)
self.n_in,self.n_out = units
self.no_bias = no_bias
if(activation is None):
self.activation = 'sigmoid'
else:
self.activation = activation
if(eur):
if(self.activation =='sigmoid'):
self.W = tf.Variable(tf.random_uniform(units,minval=(-4*(6.0/(self.n_in+self.n_out))**0.5),maxval=(4*(6.0/(self.n_in+self.n_out))**0.5)),name="W")
elif(self.activation == "leaky_relu6" or self.activation == 'relu' or self.activation == 'relu6' or self.activation == "leaky_relu"):
self.W = tf.Variable(tf.random_uniform(units,minval=0,maxval=(6.0/(self.n_in+self.n_out))**0.5),name="W")
elif(self.activation == 'tanh'):
self.W = tf.Variable(tf.random_uniform(units,minval=(-(6.0/(self.n_in+self.n_out))**0.5),maxval=((6.0/(self.n_in+self.n_out))**0.5)),name="W")
else:
self.W = tf.Variable(tf.truncated_normal(units,mean=mean,stddev=std),name="W")
else:
self.W = tf.Variable(tf.truncated_normal(units,mean=mean,stddev=std),name="W")
if(no_bias):
self.b = None
else:
self.b = tf.Variable(tf.zeros([units[1]]),name="b")
def testNoCSE(self):
with self.test_session(use_gpu=True):
shape = [2, 3, 4]
rnd1 = tf.truncated_normal(shape, 0.0, 1.0, tf.float32)
rnd2 = tf.truncated_normal(shape, 0.0, 1.0, tf.float32)
diff = rnd2 - rnd1
self.assertTrue(np.linalg.norm(diff.eval()) > 0.1)
def __init__(self, input_size, num_hidden, minibatch_size, name='lstmcell'):
"""
Constructs an LSTM Cell
Parameters:
----------
input_size: int
the size of the single input vector to the cell
num_hidden: int
the number of hidden nodes in the cell
minibatch_size: int
the number of the input vectors in the input matrix
"""
LSTMCell.created_count += 1
self.id = name + '_' + str(LSTMCell.created_count) if name == 'lstmcell' else name
self.input_size = input_size
self.num_hidden = num_hidden
self.minibatch_size = minibatch_size
self.input_weights = tf.Variable(tf.truncated_normal([self.input_size, self.num_hidden * 4], -0.1, 0.1), name=self.id + '_wi')
self.output_weights = tf.Variable(tf.truncated_normal([self.num_hidden, self.num_hidden * 4], -0.1, 0.1), name=self.id + '_wo')
self.bias = tf.Variable(tf.zeros([self.num_hidden * 4]), name=self.id + '_b')
self.prev_output = tf.Variable(tf.zeros([self.minibatch_size, self.num_hidden]), trainable=False, name=self.id+'_o')
self.prev_state = tf.Variable(tf.zeros([self.minibatch_size, self.num_hidden]), trainable=False, name=self.id+'_s')
def inference(images, hidden1_units):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
hidden1_units: Size of the first hidden layer.
hidden2_units: Size of the second hidden layer.
Returns:
softmax_linear: Output tensor with the computed logits.
"""
# Hidden 1
with tf.name_scope('hidden1'):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units],
stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights')
biases = tf.Variable(tf.zeros([hidden1_units]),
name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Hidden 2
# Linear
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(hidden1_units))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
logits = tf.matmul(hidden1, weights) + biases
return logits
def fc_layers(self):
# fc1
with tf.name_scope('fc1') as scope:
shape = int(np.prod(self.pool5.get_shape()[1:]))
fc1w = tf.Variable(tf.truncated_normal([shape, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
pool5_flat = tf.reshape(self.pool5, [-1, shape])
fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b)
self.fc1 = tf.nn.relu(fc1l)
self.parameters += [fc1w, fc1b]
# fc2
with tf.name_scope('fc2') as scope:
fc2w = tf.Variable(tf.truncated_normal([4096, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b)
self.fc2 = tf.nn.relu(fc2l)
self.parameters += [fc2w, fc2b]
# fc3
with tf.name_scope('fc3') as scope:
fc3w = tf.Variable(tf.truncated_normal([4096, 1000],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc3b = tf.Variable(tf.constant(1.0, shape=[1000], dtype=tf.float32),
trainable=True, name='biases')
self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
self.parameters += [fc3w, fc3b]
def create_graph(self):
with self.__graph.as_default():
self.__featurePlaceHolder = tf.placeholder(dtype=tf.int32, shape=[None, self.__window_size * 2])
self.__labelPlaceHolder = tf.placeholder(dtype=tf.int32, shape=[None, 1])
onehot_lookup_tables = tf.Variable(
initial_value=tf.truncated_normal(shape=[self.__vocabulary_size, self.__embedding_size])
)
embedding = tf.nn.embedding_lookup(params=onehot_lookup_tables, ids = self.__featurePlaceHolder)
projection_out = tf.reduce_mean(embedding, axis=1)
softmax_weight = tf.Variable(initial_value=tf.truncated_normal(
shape=[self.__vocabulary_size, self.__embedding_size]
))
softmax_biases = tf.Variable(initial_value=tf.zeros([self.__vocabulary_size]))
sampled_loss_per_batch = tf.nn.sampled_softmax_loss(
weights=softmax_weight,
biases=softmax_biases,
inputs=projection_out,
labels=self.__labelPlaceHolder,
num_sampled=self.__num_sampled,
num_classes=self.__vocabulary_size
)
self.__loss = tf.reduce_mean(sampled_loss_per_batch)
self.__optimizer = tf.train.AdagradOptimizer(1.0).minimize(self.__loss)
norm = tf.sqrt(tf.reduce_sum(tf.square(onehot_lookup_tables), 1, keep_dims=True))
self.__normalized_embedding = onehot_lookup_tables / norm
def runNN (train_x, train_y, test_x, test_y, numHidden):
print "NN({})".format(numHidden)
session = tf.InteractiveSession()
x = tf.placeholder("float", shape=[None, train_x.shape[1]])
y_ = tf.placeholder("float", shape=[None, 2])
W1 = tf.Variable(tf.truncated_normal([train_x.shape[1],numHidden], stddev=0.01))
b1 = tf.Variable(tf.truncated_normal([numHidden], stddev=0.01))
W2 = tf.Variable(tf.truncated_normal([numHidden,2], stddev=0.01))
b2 = tf.Variable(tf.truncated_normal([2], stddev=0.01))
z = tf.nn.relu(tf.matmul(x,W1) + b1)
y = tf.nn.softmax(tf.matmul(z,W2) + b2)
cross_entropy = -tf.reduce_sum(y_*tf.log(tf.clip_by_value(y,1e-10,1.0)))
#cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.MomentumOptimizer(learning_rate=.001, momentum=0.1).minimize(cross_entropy)
#train_step = tf.train.AdamOptimizer(learning_rate=.01).minimize(cross_entropy)
session.run(tf.initialize_all_variables())
for i in range(NUM_EPOCHS):
offset = i*BATCH_SIZE % (train_x.shape[0] - BATCH_SIZE)
train_step.run({x: train_x[offset:offset+BATCH_SIZE, :], y_: makeLabels(train_y[offset:offset+BATCH_SIZE])})
if i % 100 == 0:
util.showProgress(cross_entropy, x, y, y_, test_x, test_y)
session.close()
def inference(images,hidden1_units,hidden2_units):
"""建立前馈神经网络模型
Args:
images:输入图像数据
hidden1_units:第一个隐藏层的神经元数目
hidden2_units:第二个隐藏层 的神经元数目
returns:
softmax_linear:输出张量为计算后的结果
"""
#隐藏层1
with tf.name_scope('hidden1'):
weights = tf.Variable(tf.truncated_normal([IMAGE_PIXELS,hidden1_units],stddev=1.0/math.sqrt(float(IMAGE_PIXELS))),name='weights')#?
biases = tf.Variable(tf.zeros([hidden1_units]),name='biases')
hidden1 = tf.nn.relu(tf.matmul(images,weights)+biases)
#隐藏层2
with tf.name_scope('hidden2'):
weights = tf.Variable(tf.truncated_normal([hidden1_units,hidden2_units],stddev=1.0/math.sqrt(float(hidden1_units))),name='weights')
biases = tf.Variable(tf.zeros([hidden2_units]),name='biases')
hidden2 = tf.nn.relu(tf.matmul(hidden1,weights)+biases)
#线性输出层
with tf.name_scope('softmax_linear'):
weights = tf.Variable(tf.truncated_normal([hidden2_units,NUM_CLASSES]),name='biases')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),name='biases')
logits = tf.matmul(hidden2,weights) + biases
return logits
def autoencoder(d,c=5,tied_weights=False):
'''
An autoencoder network with one hidden layer (containing the encoding),
and sigmoid activation functions.
Args:
d: dimension of input.
c: dimension of code.
tied_weights: True if w1^T=w2
Returns:
Dictionary containing input placeholder Tensor and loss Variable
Raises:
'''
inputs = tf.placeholder(tf.float32, shape=[None,d], name='input')
w1 = tf.Variable(tf.truncated_normal([d,c], stddev=1.0/math.sqrt(d)))
b1 = tf.Variable(tf.zeros([c]))
w2 = tf.Variable(tf.truncated_normal([c,d], stddev=1.0/math.sqrt(c)))
# TODO: Implement tied weights
b2 = tf.Variable(tf.zeros([d]))
code = tf.nn.sigmoid(tf.matmul(inputs, w1)+b1, name='encoding')
reconstruction = tf.nn.sigmoid(tf.matmul(code, w2)+b2, name='reconstruction')
loss = tf.reduce_mean(tf.square(reconstruction - inputs))
tf.scalar_summary('loss', loss)
return {'inputs': inputs, 'loss': loss}
def drawGraph(self, n_row, n_latent, n_col):
with tf.name_scope('matDecomp'):
self._p = tf.placeholder(tf.float32, shape=[None, n_col])
self._c = tf.placeholder(tf.float32, shape=[None, n_col])
self._lambda = tf.placeholder(tf.float32)
self._index = tf.placeholder(tf.float32, shape=[None, n_row])
self._A = tf.Variable(tf.truncated_normal([n_row, n_latent]))
self._B = tf.Variable(tf.truncated_normal([n_latent, n_col]))
self._h = tf.matmul(tf.matmul(self._index, self._A), self._B)
weighted_loss = tf.reduce_mean(tf.mul(self._c, tf.squared_difference(self._p, self._h)))
self._weighted_loss = weighted_loss
l2_A = tf.reduce_sum(tf.square(self._A))
l2_B = tf.reduce_sum(tf.square(self._B))
n_w = tf.constant(n_row * n_latent + n_latent * n_col, tf.float32)
l2 = tf.truediv(tf.add(l2_A, l2_B), n_w)
reg_term = tf.mul(self._lambda, l2)
self._loss = tf.add(weighted_loss, reg_term)
self._mask = tf.placeholder(tf.float32, shape=[n_row, n_col])
one = tf.constant(1, tf.float32)
pred = tf.cast(tf.greater_equal(tf.matmul(self._A, self._B), one), tf.float32)
cor = tf.mul(tf.cast(tf.equal(pred, self._p), tf.float32), self._c)
self._vali_err = tf.reduce_sum(tf.mul(cor, self._mask))
self._saver = tf.train.Saver([v for v in tf.all_variables() if v.name.find('matDecomp') != -1])
tf.scalar_summary('training_weighted_loss_l2', self._loss)
tf.scalar_summary('validation_weighted_loss', self._weighted_loss)
merged = tf.merge_all_summaries()
def inference(images):
"""
Build the MNIST model
"""
# Hidden 1
with tf.name_scope('hidden1'):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, LAYER_SIZE],
stddev= 1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights')
biases = tf.Variable(tf.zeros([LAYER_SIZE]),
name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Add summary ops to collect data
tf.histogram_summary('weights', weights)
tf.histogram_summary('biases', biases)
# Output Layer - is this correct? does this layer have any weights?
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([LAYER_SIZE, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(LAYER_SIZE))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
logits = logSoftMax(tf.matmul(hidden1, weights) + biases)
return logits
def _intermediate_controller(self, register_output):
"""
Trainable model with a, b, c, and f outputs.
"""
inputs = tf.transpose(self._interpret_register(register_output),
name="interpreted_registers")
with tf.name_scope("h1"):
w = tf.Variable(tf.truncated_normal([self.R_num_registers,
self.num_h1_units],
stddev=1.0 / math.sqrt(float(self.R_num_registers))),
name="w")
b = tf.Variable(tf.zeros([self.num_h1_units]),
name="b")
h1 = tf.nn.relu(tf.matmul(inputs, w) + b, name="h1")
with tf.name_scope("a"):
w = tf.Variable(tf.truncated_normal(
[self.num_h1_units,
(self.R_num_registers + self.Q_num_modules)],
stddev=1.0 / math.sqrt(float(self.num_h1_units))),
name="w")
b = tf.Variable(tf.zeros([(self.R_num_registers + \
self.Q_num_modules)]),
name="b")
logit_a = tf.matmul(h1, w) + b
with tf.name_scope("b"):
w = tf.Variable(tf.truncated_normal(
[self.num_h1_units,
(self.R_num_registers + self.Q_num_modules)],
stddev=1.0 / math.sqrt(float(self.num_h1_units))),
name="w")
b = tf.Variable(tf.zeros([(self.R_num_registers + \
self.Q_num_modules)]),
name="b")
logit_b = tf.matmul(h1, w) + b
with tf.name_scope("c"):
w = tf.Variable(tf.truncated_normal(
[self.num_h1_units,
(self.R_num_registers + self.Q_num_modules)],
stddev=1.0 / math.sqrt(float(self.num_h1_units))),
name="w")
b = tf.Variable(tf.zeros([(self.R_num_registers + \
self.Q_num_modules)]),
name="b")
logit_c = tf.matmul(h1, w) + b
with tf.name_scope("f"):
w = tf.Variable(tf.truncated_normal(
[self.num_h1_units, 1],
stddev=1.0 / math.sqrt(float(self.num_h1_units))),
name="w")
b = tf.Variable(tf.zeros([1]),
name="b")
logit_x = tf.matmul(h1, w) + b
f = tf.nn.sigmoid(logit_x)
return logit_a, logit_b, logit_c, f
def create_subsample_layers():
print('Defining parameters ...')
for op in conv_ops:
if 'fulcon' in op:
#we don't create weights biases for fully connected layers because we need to calc the
#fan_out of the last convolution/pooling (subsampling) layer
#as that's gonna be fan_in for the 1st hidden layer
break
if 'conv' in op:
print('\tDefining weights and biases for %s (weights:%s)'%(op,hyparams[op]['weights']))
print('\t\tWeights:%s'%hyparams[op]['weights'])
print('\t\tBias:%d'%hyparams[op]['weights'][3])
weights[op]=tf.Variable(
tf.truncated_normal(hyparams[op]['weights'],
stddev=2./min(5,hyparams[op]['weights'][0])
)
)
biases[op] = tf.Variable(tf.constant(np.random.random()*0.001,shape=[hyparams[op]['weights'][3]]))
if 'incept' in op:
print('\n\tDefining the weights and biases for the Incept Module')
inc_hyparams = hyparams[op]
for k,v in inc_hyparams.items():
if 'conv' in k:
w_key = op+'_'+k
print('\t\tParameters for %s'%w_key)
print('\t\t\tWeights:%s'%inc_hyparams[k]['weights'])
print('\t\t\tBias:%d'%inc_hyparams[k]['weights'][3])
weights[w_key] = tf.Variable(
tf.truncated_normal(inc_hyparams[k]['weights'],
stddev=2./min(5,inc_hyparams[k]['weights'][0])
)
)
biases[w_key] = tf.Variable(tf.constant(np.random.random()*0.0,shape=[inc_hyparams[k]['weights'][3]]))
def get_initilizations_standard_NN(init_type,dims,mu,std,b_init,X_train,Y_train,train_S_type='multiple_S'):
#def get_initilizations_standard_NN(args):
if init_type=='truncated_normal':
inits_W=[None]
inits_b=[None]
nb_hidden_layers=len(dims)-1
for l in range(1,nb_hidden_layers):
#inits_W.append( tf.truncated_normal(shape=[dims[l-1],dims[l]], mean=mu[l], stddev=std[l], dtype=tf.float32) )
inits_W.append( tf.truncated_normal(shape=[dims[l-1],dims[l]], mean=mu[l], stddev=std[l], dtype=tf.float32) )
inits_b.append( tf.constant( b_init[l], shape=[ dims[l] ], dtype=tf.float32 ) )
l=len(dims)-1
#print( [dims[l-1],dims[l]])
inits_C=[ tf.truncated_normal(shape=[dims[l-1],dims[l]], mean=mu[l], stddev=std[l], dtype=tf.float32) ]
# elif init_type=='data_init':
# X_train=X_train
# pass
elif init_type=='xavier':
inits_W=[None]
inits_b=[None]
nb_hidden_layers=len(dims)-1
for l in range(1,nb_hidden_layers):
inits_W.append( tf.contrib.layers.xavier_initializer(dtype=tf.float32) )
inits_b.append( tf.constant( b_init[l], shape=[dims[l]], dtype=tf.float32 ) )
l=len(dims)-1
#print( 'dims inits_C: ',[dims[l-1],dims[l]] )
print('l ', l)
print( '+++> std mu for inits_C: ',[mu[l],std[l]] )
inits_C=[ tf.truncated_normal(shape=[dims[l-1],dims[l]], mean=mu[l], stddev=std[l], dtype=tf.float32) ]
else:
raise ValueError('Need to use INIT library properly')
return (inits_C,inits_W,inits_b)
def create_new_conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, name):
# setup the filter input shape for tf.nn.conv_2d
conv_filt_shape = [filter_shape[0], filter_shape[1], num_input_channels, num_filters]
# initialise weights and bias for the filter
weights = tf.Variable(tf.truncated_normal(conv_filt_shape, stddev=0.03), name=name+'_W')
bias = tf.Variable(tf.truncated_normal([num_filters]), name=name+'_b')
# setup the convolutional layer operation
out_layer = tf.nn.conv2d(input_data, weights, [1, 1, 1, 1], padding='SAME')
# add the bias
out_layer += bias
# apply a ReLU non-linear activation
out_layer = tf.nn.relu(out_layer)
# now perform max pooling
# ksize is the argument which defines the size of the max pooling window (i.e. the area over which the maximum is
# calculated). It must be 4D to match the convolution - in this case, for each image we want to use a 2 x 2 area
# applied to each channel
ksize = [1, pool_shape[0], pool_shape[1], 1]
# strides defines how the max pooling area moves through the image - a stride of 2 in the x direction will lead to
# max pooling areas starting at x=0, x=2, x=4 etc. through your image. If the stride is 1, we will get max pooling
# overlapping previous max pooling areas (and no reduction in the number of parameters). In this case, we want
# to do strides of 2 in the x and y directions.
strides = [1, 2, 2, 1]
out_layer = tf.nn.max_pool(out_layer, ksize=ksize, strides=strides, padding='SAME')
return out_layer
def quadratic_feedforward(input_layer, layers: Sequence[int], has_activation: bool = False):
prior_node_count = input_layer.get_shape()[1].value
for i in range(len(layers)):
last_layer = i == len(layers) - 1
name = 'output_layer' if last_layer else 'layer_' + str(i)
node_count = layers[i]
with tf.name_scope(name):
fw = tf.Variable(
tf.truncated_normal([prior_node_count, node_count],
stddev=1.0 / math.sqrt(float(1))),
name='fw')
fb = tf.Variable(tf.zeros([node_count]),
name='fb')
gw = tf.Variable(
tf.truncated_normal([prior_node_count, node_count],
stddev=1.0 / math.sqrt(float(1))),
name='gw')
gb = tf.Variable(tf.zeros([node_count]),
name='gb')
input_layer = tf.mul(tf.matmul(input_layer, fw) + fb, tf.matmul(input_layer, gw) + gb)
if has_activation and not last_layer:
input_layer = tf.nn.relu(input_layer)
prior_node_count = node_count
return input_layer
def setup(self):
# INPUT
self.input = tf.placeholder(tf.float32, [None,256,256,3])
self.layer_shapes = [ [256, 256, 3], [128, 128, 16], [64, 64, 32], [32, 32, 32], [16, 16, 64] ]
# Convolution 1
self.w1 = tf.Variable(tf.truncated_normal([15,15,self.layer_shapes[0][2],self.layer_shapes[1][2]], stddev=0.1))
self.b1 = tf.Variable(tf.zeros([16]))
self.conv_1 = tf.nn.tanh(tf.nn.conv2d(self.input, self.w1, strides=[1, 2, 2, 1], padding='SAME')+self.b1)
# Convolution 2
self.w2 = tf.Variable(tf.truncated_normal([7,7,self.layer_shapes[1][2],self.layer_shapes[2][2]], stddev=0.1))
self.b2 = tf.Variable(tf.zeros([32]))
self.conv_2 = tf.nn.tanh(tf.nn.conv2d(self.conv_1, self.w2, strides=[1, 2, 2, 1], padding='SAME')+self.b2)
# Convolution 3
self.w3 = tf.Variable(tf.truncated_normal([5,5,self.layer_shapes[2][2],self.layer_shapes[3][2]], stddev=0.1))
self.b3 = tf.Variable(tf.zeros([32]))
self.conv_3 = tf.nn.tanh(tf.nn.conv2d(self.conv_2, self.w3, strides=[1, 2, 2, 1], padding='SAME')+self.b3)
# Convolution 4
self.w4 = tf.Variable(tf.truncated_normal([5,5,self.layer_shapes[3][2],self.layer_shapes[4][2]], stddev=0.1))
self.b4 = tf.Variable(tf.zeros([64]))
self.conv_4 = tf.nn.tanh(tf.nn.conv2d(self.conv_3, self.w4, strides=[1, 2, 2, 1], padding='SAME')+self.b4)
self.outputs = [self.conv_1, self.conv_2, self.conv_3, self.conv_4]
self.trainables = [ [self.w1, self.b1], [self.w2, self.b2], [self.w3, self.b3], [self.w4, self.b4] ]
return self.outputs
def encode(images,texts):
#img_hidden
with tf.name_scope('img_hidden'):
weights = tf.Variable(
tf.truncated_normal([const.INP_IMAGE_DIM, const.HIDDEN_IMG_DIM],
stddev=1.0 / math.sqrt(float(const.HIDDEN_IMG_DIM))),
name='weights')
biases = tf.Variable(tf.zeros([const.HIDDEN_IMG_DIM]),
name='biases')
hidden_img = tf.nn.relu(tf.matmul(images, weights) + biases)
# Text_Hidden
with tf.name_scope('txt_hidden'):
weights = tf.Variable(
tf.truncated_normal([const.INP_TXT_DIM, const.HIDDEN_TXT_DIM],
stddev=1.0 / math.sqrt(float(const.HIDDEN_TXT_DIM))),
name='weights')
biases = tf.Variable(tf.zeros([const.HIDDEN_TXT_DIM]),
name='biases')
hidden_txt = tf.nn.relu(tf.matmul(texts, weights) + biases)
# Linear
with tf.name_scope('join_layer'):
hidden_con = tf.concat([hidden_img,hidden_txt],axis=-1,name="hidden_con");
weights = tf.Variable(
tf.truncated_normal([const.HIDDEN_CON_DIM, const.SHARED_DIM],
stddev=1.0 / math.sqrt(float(const.SHARED_DIM))),
name='weights')
biases = tf.Variable(tf.zeros([const.SHARED_DIM]),
name='biases')
representive = tf.matmul(hidden_con, weights) + biases
return representive
def inference(images, hidden1_units, hidden2_units):
with tf.name_scope("hidden1"):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units], stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name="weights",
)
biases = tf.Variable(tf.zeros([hidden1_units]), name="biases")
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
with tf.name_scope("hidden2"):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, hidden2_units], stddev=1.0 / math.sqrt(float(hidden1_units))),
name="weights",
)
biases = tf.Variable(tf.zeros([hidden2_units]), name="biases")
hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases)
with tf.name_scope("softmax_linear"):
weights = tf.Variable(
tf.truncated_normal([hidden2_units, NUM_CLASSES], stddev=1.0 / math.sqrt(float(hidden2_units))),
name="weights",
)
biases = tf.Variable(tf.zeros([NUM_CLASSES]), name="biases")
logits = tf.matmul(hidden2, weights) + biases
return logits
def init_model(self):
# input layer (8 x 8)
self.x = tf.placeholder(tf.float32, [None, 8, 8])
# flatten (64)
x_flat = tf.reshape(self.x, [-1, 64])
# fully connected layer (32)
W_fc1 = tf.Variable(tf.truncated_normal([64, 64], stddev=0.01))
b_fc1 = tf.Variable(tf.zeros([64]))
h_fc1 = tf.nn.relu(tf.matmul(x_flat, W_fc1) + b_fc1)
# output layer (n_actions)
W_out = tf.Variable(tf.truncated_normal([64, self.n_actions], stddev=0.01))
b_out = tf.Variable(tf.zeros([self.n_actions]))
self.y = tf.matmul(h_fc1, W_out) + b_out
# loss function
self.y_ = tf.placeholder(tf.float32, [None, self.n_actions])
self.loss = tf.reduce_mean(tf.square(self.y_ - self.y))
# train operation
optimizer = tf.train.RMSPropOptimizer(self.learning_rate)
self.training = optimizer.minimize(self.loss)
# saver
self.saver = tf.train.Saver()
# session
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def inference(images):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
hidden1_units: Size of the first hidden layer.
hidden2_units: Size of the second hidden layer.
Returns:
softmax_linear: Output tensor with the computed logits.
"""
# Hidden 1
with tf.name_scope('conv1'):
images=tf.reshape(images,[-1,32,32,3])
conv1_w=tf.Variable(tf.truncated_normal([5,5,3,8]),name='weights')
conv1_b=tf.Variable(tf.zeros([8]),name='bias')
conv1=tf.nn.relu(tf.nn.conv2d(images,conv1_w,[1,1,1,1],'VALID')+conv1_b)
with tf.name_scope('conv2'):
conv2_w = tf.Variable(tf.truncated_normal([5, 5, 8, 8]), name='weights')
conv2_b = tf.Variable(tf.zeros([8]), name='bias')
conv2 = tf.nn.relu(tf.nn.conv2d(conv1, conv2_w, strides=[1, 1, 1, 1],padding='VALID') + conv2_b)
conv2_flatten=tf.reshape(conv2,[-1,24*24*8])
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([24*24*8, NUM_CLASSES]),name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),name='biases')
logits = tf.matmul(conv2_flatten, weights) + biases
return logits
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
bias1 = tf.Variable(tf.constant(0.0, shape=[LAYER1_NODE]))
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
bias2 = tf.Variable(tf.constant(0.0, shape=[OUTPUT_NODE]))
layer1 = tf.nn.relu(tf.matmul(x, weights1) + bias1)
y = tf.matmul(layer1, weights2) + bias2
global_step = tf.Variable(0, trainable=False)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
loss = tf.reduce_mean(cross_entropy)
train_op=tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss, global_step=global_step)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as sess:
tf.initialize_all_variables().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))
xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})
test_acc = sess.run(accuracy, feed_dict=test_feed)
print("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc))
def __init__(self, optimizer, categories, num_of_terms, num_of_hidden_nodes):
self.optimizer = optimizer
self.categories = categories
self.num_of_categories = len(self.categories)
self.num_of_terms = num_of_terms
self.num_of_hidden_nodes = num_of_hidden_nodes
self.input_ph = tf.placeholder(tf.float32, [None, self.num_of_terms], name="input")
self.supervisor_ph = tf.placeholder(tf.float32, [None, self.num_of_categories], name="supervisor")
with tf.name_scope("inference") as scope:
weight1_var = tf.Variable(tf.truncated_normal([self.num_of_terms, self.num_of_hidden_nodes], stddev=0.1), name="weight1")
weight2_var = tf.Variable(tf.truncated_normal([self.num_of_hidden_nodes, self.num_of_categories], stddev=0.1), name="weight2")
bias1_var = tf.Variable(tf.zeros([self.num_of_hidden_nodes]), name="bias1")
bias2_var = tf.Variable(tf.zeros([self.num_of_categories]), name="bias2")
hidden_op = tf.nn.relu(tf.matmul(self.input_ph, weight1_var) + bias1_var)
self.output_op = tf.nn.softmax(tf.matmul(hidden_op, weight2_var) + bias2_var)
with tf.name_scope("loss") as scope:
cross_entropy = -tf.reduce_sum(self.supervisor_ph * tf.log(self.output_op))
l2_sqr = tf.nn.l2_loss(weight1_var) + tf.nn.l2_loss(weight2_var)
lambda_2 = 0.01
self.loss_op = cross_entropy + lambda_2 * l2_sqr
tf.scalar_summary("loss", self.loss_op)
with tf.name_scope("training") as scope:
self.training_op = self.optimizer.minimize(self.loss_op)
with tf.name_scope("accuracy") as scope:
correct_prediction = tf.equal(tf.argmax(self.output_op, 1), tf.argmax(self.supervisor_ph, 1))
self.accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, "float"))
tf.scalar_summary("accuracy", self.accuracy_op)
self.summary_op = tf.merge_all_summaries()
def inference(images, hidden1_units):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
hidden1_units: Size of the first hidden layer.
Returns:
softmax_linear: Output tensor with the computed logits.
"""
# Hidden 1
with tf.name_scope('hidden1'):
'''
A Variable is a modifiable tensor that lives in TensorFlow's graph of interacting operations.
It can be used and even modified by the computation. For machine learning applications,
one generally has the model parameters be Variables.
'''
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units], stddev=1.0 / np.sqrt(float(IMAGE_PIXELS))), name='weights')
biases = tf.Variable(tf.zeros([hidden1_units]), name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Linear
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, NUM_CLASSES], stddev=1.0 / np.sqrt(float(hidden1_units))), name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]), name='biases')
logits = tf.matmul(hidden1, weights) + biases
return logits
def mnist_inference(images, hidden1_units):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder.
hidden1_units: Size of the first hidden layer.
Returns:
logits: Output tensor with the computed logits.
"""
# Hidden 1
with tf.name_scope('hidden1'):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units],
stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights')
biases = tf.Variable(tf.zeros([hidden1_units]),
name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Linear
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(hidden1_units))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
logits = tf.matmul(hidden1, weights) + biases
# Uncomment the following line to see what we have constructed.
# tf.train.write_graph(tf.get_default_graph().as_graph_def(),
# "/tmp", "inference.pbtxt", as_text=True)
return logits
def __init__(self, embedding_length):
self._embedding_length = embedding_length
self._named_tensors = {}
for n in xrange(10):
# Note: the examples only have the numbers 0 through 9 as terminal nodes.
name = 'terminal_' + str(n)
self._named_tensors[name] = tf.Variable(
tf.truncated_normal([embedding_length],
dtype=tf.float32,
stddev=1),
name=name)
self._combiner_weights = {}
self._loom_ops = {}
for name in calculator_pb2.CalculatorExpression.OpCode.keys():
weights_var = tf.Variable(
tf.truncated_normal([2 * embedding_length, embedding_length],
dtype=tf.float32,
stddev=1),
name=name)
self._combiner_weights[name] = weights_var
self._loom_ops[name] = CombineLoomOp(2, embedding_length, weights_var)
self._loom = loom.Loom(
named_tensors=self._named_tensors,
named_ops=self._loom_ops)
self._output = self._loom.output_tensor(
loom.TypeShape('float32', [embedding_length]))
def get_inference(images_ph, dropout_keep_prob_ph):
#subtract average image
with tf.variable_scope('centering') as scope:
mean = tf.constant(vgg.average_image, dtype=tf.float32, name='avg_image')
images_ph = tf.sub(images_ph, mean, name='subtract_avg')
#get layers from vgg19
vgg_layers = vgg.get_VGG_layers(images_ph, dropout_keep_prob_ph, train_fc_layers=True)
#################################################
### Add more layers for semantic segmentation ###
#################################################
# convolution on top of pool4 to 21 chammenls (to make coarse predictions)
with tf.variable_scope('conv9') as scope:
conv9 = conv_layer(vgg_layers['pool4'], 21, 1, 'conv9')
# convolution on top of conv7 (fc7) to 21 chammenls (to make coarse predictions)
with tf.variable_scope('conv8') as scope:
conv8 = conv_layer(vgg_layers['dropout2'], 21, 1, 'conv8')
# 2x upsampling from last layer
with tf.variable_scope('deconv1') as scope:
shape = tf.shape(conv8)
out_shape = tf.pack([shape[0], shape[1]*2, shape[2]*2, 21])
weights = tf.Variable(tf.truncated_normal(mean=MEAN, stddev=0.1, shape=(4, 4, 21, 21)), name='weights')
deconv1 = tf.nn.conv2d_transpose( value=conv8,
filter=weights,
output_shape=out_shape,
strides=(1, 2, 2, 1),
padding='SAME',
name='deconv1')
# slice 2x upsampled tensor in the last layer to fit pool4
shape = tf.shape(conv9)
size = tf.pack([-1, shape[1], shape[2], -1])
deconv1 = tf.slice(deconv1, begin=[0,0,0,0], size=size, name="deconv1_slice")
# combine preductions from last layer and pool4
with tf.variable_scope('combined_pred') as scope:
combined_pred = tf.add(deconv1, conv9, name="combined_pred")
# 16x upsampling
with tf.variable_scope('deconv2') as scope:
shape = tf.shape(combined_pred)
out_shape = tf.pack([shape[0], shape[1]*16, shape[2]*16, 21])
weights = tf.Variable(tf.truncated_normal(mean=MEAN, stddev=0.1, shape=(32, 32, 21, 21)), name='weights')
deconv2 = tf.nn.conv2d_transpose(value=combined_pred,
filter=weights,
output_shape=out_shape,
strides=(1, 16, 16, 1),
padding='SAME',
name='deconv2')
# slice upsampled tensor to original shape
orig_shape = tf.shape(images_ph)
size = tf.pack([-1, orig_shape[1], orig_shape[2], -1])
logits = tf.slice(deconv2, begin=[0,0,0,0], size=size, name='logits')
return logits
def __init__(self, ob_dim):
self.sess = tf.Session()
self.sess.__enter__()
self.sy_obs = tf.placeholder(tf.float32, [None,ob_dim])
self.sy_vals = tf.placeholder(tf.float32, [None])
beta = 0.0000
num_h = 5
w1 = tf.Variable(tf.truncated_normal([ob_dim,num_h], stddev=1.0/ob_dim**0.5))
b1 = tf.Variable(tf.zeros(num_h))
h1 = tf.nn.relu(tf.matmul(self.sy_obs,w1)+b1)
w2 = tf.Variable(tf.truncated_normal([num_h,num_h], stddev=1.0/num_h**0.5))
b2 = tf.Variable(tf.zeros(num_h))
h2 = tf.nn.relu(tf.matmul(h1,w2)+b2)
#w2 = tf.Variable(tf.truncated_normal([num_h,1], stddev=1.0/num_h**0.5))
#b2 = tf.Variable(tf.zeros(1))
#self.pred_vals = tf.reshape(tf.matmul(h1,w2)+b2,shape=[-1])
w3 = tf.Variable(tf.truncated_normal([num_h,1], stddev=1.0/num_h**0.5))
b3 = tf.Variable(tf.zeros(1))
self.pred_vals = tf.reshape(tf.matmul(h2,w3)+b3,shape=[-1])
loss = tf.reduce_mean(self.sy_vals-self.pred_vals)
reg = tf.nn.l2_loss(w1)+tf.nn.l2_loss(w2)
loss = tf.reduce_mean(loss+beta*reg)
self.train_step = tf.train.AdamOptimizer().minimize(loss)
init = tf.global_variables_initializer()
self.sess.run(init)
def inference(images):
parameters = []
# 第一个卷积层, scope可以将变量名称之前自动加上conv1，比如conv1/weights
with tf.name_scope('conv1') as scope:
kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 64],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(images,
kernel,
strides=[1, 4, 4, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True,
name='biases')
add_conv_biases = tf.add(conv, biases)
conv_relu = tf.nn.relu(add_conv_biases, name=scope)
print_activations(conv_relu)
parameters += [kernel, biases]
# 池化层，最大池化
pool1 = tf.nn.max_pool(conv_relu,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool1')
print_activations(pool1)
# conv2
with tf.name_scope('conv2') as scope:
kernel = tf.Variable(tf.truncated_normal([5, 5, 64, 192],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[192], dtype=tf.float32),
trainable=True,
name='biases')
bias = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv2)
# pool2
pool2 = tf.nn.max_pool(conv2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool2')
print_activations(pool2)
# conv3
with tf.name_scope('conv3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 192, 384],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[384], dtype=tf.float32),
trainable=True,
name='biases')
bias = tf.nn.bias_add(conv, biases)
conv3 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv3)
# conv4
with tf.name_scope('conv4') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True,
name='biases')
bias = tf.nn.bias_add(conv, biases)
conv4 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv4)
# conv5
with tf.name_scope('conv5') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True,
name='biases')
bias = tf.nn.bias_add(conv, biases)
conv5 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv5)
# pool5
pool5 = tf.nn.max_pool(conv5,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool5')
print_activations(pool5)
return pool5, parameters
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def inference(images):
parameters = []
# conv1
#该语句用于将scope内生成的Variable自动命名为conv1/xxx，便于区分不同卷积层之间的组件
with tf.name_scope('conv1') as scope:
kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 64], dtype=tf.float32,stddev=1e-1), name='weights')
conv = tf.nn.conv2d(images, kernel, [1, 4, 4, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),trainable=True, name='biases')
bias = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(bias, name=scope)
print_activations(conv1)
#记录所有需要训练的参数
parameters += [kernel, biases]
# pool1
lrn1 = tf.nn.lrn(conv1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='lrn1')
pool1 = tf.nn.max_pool(lrn1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool1')
print_activations(pool1)
# conv2
with tf.name_scope('conv2') as scope:
kernel = tf.Variable(tf.truncated_normal([5, 5, 64, 192], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[192], dtype=tf.float32),
trainable=True, name='biases')
bias = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv2)
# pool2
lrn2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='lrn2')
pool2 = tf.nn.max_pool(lrn2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool2')
print_activations(pool2)
# conv3
with tf.name_scope('conv3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 192, 384],
dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[384], dtype=tf.float32),
trainable=True, name='biases')
bias = tf.nn.bias_add(conv, biases)
conv3 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv3)
# conv4
with tf.name_scope('conv4') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256],
dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases')
bias = tf.nn.bias_add(conv, biases)
conv4 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv4)
# conv5
with tf.name_scope('conv5') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases')
bias = tf.nn.bias_add(conv, biases)
conv5 = tf.nn.relu(bias, name=scope)
parameters += [kernel, biases]
print_activations(conv5)
# pool5
pool5 = tf.nn.max_pool(conv5,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pool5')
print_activations(pool5)
return pool5, parameters
def __init__(
self, sequence_length, num_classes, vocab_size,
embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0):
# Placeholders for input, output and dropout
self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x")
self.input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y")
self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
self.learning_rate = tf.placeholder(tf.float32, name="learning_rate")
# Keeping track of l2 regularization loss (optional)
l2_loss = tf.constant(0.0)
# Embedding layer
with tf.device('/cpu:0'), tf.name_scope("embedding"):
self.W = tf.Variable(
tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0), # trainable=False
name="W")
self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x) # [None, sequence_length, embedding_size]
self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1) # [None, sequence_length, embedding_size, 1]
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_chars_expanded,
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
# Maxpooling over the outputs
pooled = tf.nn.max_pool(
h,
ksize=[1, sequence_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_outputs.append(pooled)
# Combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
self.h_pool = tf.concat(pooled_outputs, 3)
self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])
# Add dropout
with tf.name_scope("dropout"):
self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
W = tf.get_variable(
"W",
shape=[num_filters_total, num_classes],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
self.predictions = tf.argmax(self.scores, 1, name="predictions")
# Calculate mean cross-entropy loss
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def __init__(self, user_num, item_num, f, user_pos_length, user_neg_length, item_pos_length, item_neg_length,
user_pos_vocab_size, user_neg_vocab_size, item_pos_vocab_size, item_neg_vocab_size, embedding_size,
filter_sizes, num_filters, n_pos_aspect, n_neg_aspect):
self.input_u_pos = tf.placeholder(tf.int32, [None, user_pos_length], name='input_u_pos')
self.input_u_neg = tf.placeholder(tf.int32, [None, user_neg_length], name='input_u_neg')
self.input_i_pos = tf.placeholder(tf.int32, [None, item_pos_length], name='input_i_pos')
self.input_i_neg = tf.placeholder(tf.int32, [None, item_neg_length], name='input_i_neg')
self.input_y = tf.placeholder(tf.float32, [None, 1], name="input_y")
self.input_uid = tf.placeholder(tf.int32, [None, 1], name="input_uid")
self.input_iid = tf.placeholder(tf.int32, [None, 1], name="input_iid")
self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
with tf.name_scope("user_pos_embedding"):
self.Wu_pos = tf.Variable(tf.random_uniform([user_pos_vocab_size, embedding_size], -1.0, 1.0), trainable=False, name='Wu_pos')
self.embedded_user_pos = tf.nn.embedding_lookup(self.Wu_pos, self.input_u_pos)
self.embedded_users_pos = tf.expand_dims(self.embedded_user_pos, -1)
with tf.name_scope("user_neg_embedding"):
self.Wu_neg = tf.Variable(tf.random_uniform([user_neg_vocab_size, embedding_size], -1.0, 1.0), trainable=False, name='Wu_neg')
self.embedded_user_neg = tf.nn.embedding_lookup(self.Wu_neg, self.input_u_neg)
self.embedded_users_neg = tf.expand_dims(self.embedded_user_neg, -1)
with tf.name_scope("item_pos_embedding"):
self.Wi_pos = tf.Variable(tf.random_uniform([item_pos_vocab_size, embedding_size], -1.0, 1.0), trainable=False, name='Wi_pos')
self.embedded_item_pos = tf.nn.embedding_lookup(self.Wi_pos, self.input_i_pos)
self.embedded_items_pos = tf.expand_dims(self.embedded_item_pos, -1)
with tf.name_scope("item_neg_embedding"):
self.Wi_neg = tf.Variable(tf.random_uniform([item_neg_vocab_size, embedding_size], -1.0, 1.0), trainable=False, name='Wi_neg')
self.embedded_item_neg = tf.nn.embedding_lookup(self.Wi_neg, self.input_i_neg)
self.embedded_items_neg = tf.expand_dims(self.embedded_item_neg, -1)
with tf.name_scope("user_latent_factors"):
self.user_Matrix = tf.Variable(tf.random_uniform([user_num, f], -1.0, 1.0), name='user_Matrix')
self.user_latent = tf.nn.embedding_lookup(self.user_Matrix, self.input_uid)
self.user_latent = tf.reshape(self.user_latent, [-1, f])
with tf.name_scope("item_latent_factors"):
self.item_Matrix = tf.Variable(tf.random_uniform([item_num, f], -1.0, 1.0), name='item_Matrix')
self.item_latent = tf.nn.embedding_lookup(self.item_Matrix, self.input_iid)
self.item_latent = tf.reshape(self.item_latent, [-1, f])
with tf.name_scope("pos_aspect_weight"):
self.pos_W = tf.Variable(tf.random_uniform([n_pos_aspect, f], -1.0, 1.0), name='pos_W')
with tf.name_scope("neg_aspect_weight"):
self.neg_W = tf.Variable(tf.random_uniform([n_neg_aspect, f], -1.0, 1.0), name='neg_W')
output_u_pos = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("user_pos_conv-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_users_pos,
W,
strides=[1, 1, embedding_size, 1],
padding="SAME",
name="conv") # batch_size * user_pos_length * 1 * num_filters
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
h1 = tf.reshape(h, [-1, user_pos_length, num_filters])
output_u_pos.append(h1)
num_filters_total = num_filters * len(filter_sizes)
self.output_u_pos_con = tf.concat(output_u_pos, 2)
self.output_u_pos_res = tf.reshape(self.output_u_pos_con, [-1, num_filters_total])
# Layer 1
Wu_pos_1 = tf.get_variable("Wu_pos_1", shape=[num_filters_total, n_pos_aspect],
initializer=tf.contrib.layers.xavier_initializer())
bu_pos_1 = tf.Variable(tf.constant(0.1, shape = [n_pos_aspect]))
self.u_pos_l1 = tf.nn.softmax(tf.nn.relu(tf.matmul(self.output_u_pos_res, Wu_pos_1) + bu_pos_1))
self.pos_asp = tf.reduce_sum(tf.reshape(self.u_pos_l1, [-1, user_pos_length, n_pos_aspect]), axis=1)
self.pos_asp_imp = tf.nn.softmax(self.pos_asp) # batch_size * n_pos_aspect
output_u_neg = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("user_neg_conv-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1,
num_filters] # [filter_height, filter_width, in_channels, out_channels]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_users_neg,
W,
strides=[1, 1, embedding_size, 1],
padding="SAME",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
h1 = tf.reshape(h, [-1, user_neg_length, num_filters])
output_u_neg.append(h1)
self.output_u_neg_con = tf.concat(output_u_neg, 2)
self.output_u_neg_res = tf.reshape(self.output_u_neg_con, [-1, num_filters_total])
# Layer 1
Wu_neg_1 = tf.get_variable("Wu_neg_1", shape=[num_filters_total, n_neg_aspect],
initializer=tf.contrib.layers.xavier_initializer())
bu_neg_1 = tf.Variable(tf.constant(0.1, shape=[n_neg_aspect]))
self.u_neg_l1 = tf.nn.softmax(tf.nn.relu(tf.matmul(self.output_u_neg_res, Wu_neg_1) + bu_neg_1))
self.neg_asp = tf.reduce_sum(tf.reshape(self.u_neg_l1, [-1, user_neg_length, n_neg_aspect]), axis=1)
self.neg_asp_imp = tf.nn.softmax(self.neg_asp) # batch_size * n_neg_aspect
neg_asp_imp_add = []
with tf.name_scope("pos2neg_imp"):
W = tf.Variable(tf.truncated_normal(shape=[f, f], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[f]), name='b')
h = tf.Variable(tf.truncated_normal(shape=[f, 1], stddev=0.1), name="h")
for i in range(n_neg_aspect):
neg_Wi = self.neg_W[i]
mul = tf.multiply(self.pos_W, neg_Wi)
rel = tf.nn.relu(tf.matmul(mul, W) + b)
attn = tf.nn.softmax(tf.matmul(rel, h), dim=0) # n_pos_aspect * 1
neg_asp_imp_i = tf.matmul(self.pos_asp_imp, attn) # batch_size * 1
neg_asp_imp_add.append(neg_asp_imp_i)
pos_asp_imp_add = []
with tf.name_scope("neg2pos_imp"):
W = tf.Variable(tf.truncated_normal(shape=[f, f], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[f]), name='b')
h = tf.Variable(tf.truncated_normal(shape=[f, 1], stddev=0.1), name="h")
for i in range(n_pos_aspect):
pos_Wi = self.pos_W[i]
mul = tf.multiply(self.neg_W, pos_Wi)
rel = tf.nn.relu(tf.matmul(mul, W) + b)
attn = tf.nn.softmax(tf.matmul(rel, h), dim=0)
pos_asp_imp_i = tf.matmul(self.neg_asp_imp, attn)
pos_asp_imp_add.append(pos_asp_imp_i)
with tf.name_scope("prediction"):
# print(self.user_latent.shape())
self.interaction = tf.multiply(self.user_latent, self.item_latent)
self.pos_asp_r = tf.matmul(self.interaction, tf.transpose(self.pos_W)) # batch_size * n_pos_asp
self.pos_imp = self.pos_asp_imp + tf.concat(pos_asp_imp_add, -1)
self.pos_r = tf.reduce_sum(tf.multiply(self.pos_asp_r, self.pos_imp), axis=-1)
self.neg_asp_r = tf.matmul(self.interaction, tf.transpose(self.neg_W))
self.neg_imp = self.neg_asp_imp + tf.concat(neg_asp_imp_add, -1)
self.neg_r = tf.reduce_sum(tf.multiply(self.neg_asp_r, self.neg_imp), axis=-1)
self.predictions = self.pos_r - self.neg_r
regularizer = layers.l2_regularizer(scale=1.0)
Var_list_1 = [Wu_pos_1, bu_pos_1, Wu_neg_1, bu_neg_1]
for i, filter_size in enumerate(filter_sizes):
Var_list_1 += tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="user_pos_conv-%s" % filter_size)
Var_list_1 += tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="user_neg_conv-%s" % filter_size)
reg_1 = layers.apply_regularization(regularizer, weights_list=Var_list_1)
Var_list_2 = []
Var_list_3 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="pos2neg_imp") \
+ tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="neg2pos_imp")
reg_3 = layers.apply_regularization(regularizer, weights_list=Var_list_3)
self.variables = Var_list_1 + Var_list_2 + Var_list_3
reg_4 = layers.apply_regularization(regularizer, weights_list=[self.user_Matrix, self.item_Matrix, self.pos_W, self.neg_W])
with tf.name_scope("loss"):
beta_1 = 1e-4
beta_2 = 0.001
losses = tf.reduce_mean(tf.square(tf.subtract(self.predictions, self.input_y)))
self.loss = losses + beta_2 * (reg_1 + reg_3 + reg_4)
with tf.name_scope("accuracy"):
self.mae = tf.reduce_mean(tf.abs(tf.subtract(self.predictions, self.input_y)))
self.accuracy = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.predictions, self.input_y))))
def add_new_weights(shape):
return tf.Variable(tf.truncated_normal(shape, stddev=0.05))
def inference(dataset, conv1_uints, conv1_kernel, conv1_stride, conv2_uints, conv3_uints, conv4_uints, fc1_uints, fc2_uints):
"""Build the model up to where it may be used for inference.
Args:
dataset: Data placeholder, from inputs()
conv1_units, conv2_uints, conv3_uints, conv4_uints: Size of the convolution layer
conv1_kernel: kernel size, which sharing same weights
conv1_stride: stride size
fc1_units, fc2_uints: Size of the fully connection layer
Returns:
softmax_re: Output tensor with the computed logits
"""
# conv1
with tf.name_scope('conv1'):
weights = tf.Variable(
tf.truncated_normal([1, conv1_kernel, 1, conv1_uints],
stddev=1.0 / math.sqrt(float(conv1_uints))),
name='weights')
biases = tf.Variable(tf.zeros(conv1_uints),
name='biases')
x_data = tf.reshape(dataset, [-1, 1, BANDS_SIZE * conv1_stride, 1]) #这里注意之后邻域变化需要修改band size的值
print(x_data.get_shape())
#conv1 = tf.nn.relu(biases + tf.nn.conv2d(x_data, weights,
#strides=[1, conv1_stride, conv1_stride, 1],
#padding='VALID'))
conv1 = tf.sigmoid(biases + tf.nn.conv2d(x_data, weights,
strides=[1, conv1_stride, conv1_stride, 1],
padding='VALID'))
print(conv1.get_shape())
# reshape
with tf.name_scope('reshape'):
reshape = tf.reshape(conv1, [-1, conv1_uints, conv1_uints, 1])
print(reshape.get_shape())
# conv2
with tf.name_scope('conv2'):
weights = tf.Variable(
tf.truncated_normal([3, 3, 1, conv2_uints],
stddev=1.0 / math.sqrt(float(conv2_uints))),
name='weights')
biases = tf.Variable(tf.zeros(conv2_uints),
name='biases')
conv2 = tf.sigmoid(biases + tf.nn.conv2d(reshape, weights,
strides=[1, 1, 1, 1],
padding='VALID'))
# mpool
with tf.name_scope('mpool1'):
mpool1 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
# conv3
with tf.name_scope('conv3'):
weights = tf.Variable(
tf.truncated_normal([3, 3, conv2_uints, conv3_uints],
stddev=1.0 / math.sqrt(float(conv3_uints))),
name='weights')
biases = tf.Variable(tf.zeros(conv3_uints),
name='biases')
conv3 = tf.sigmoid(biases + tf.nn.conv2d(mpool1, weights,
strides=[1, 1, 1, 1],
padding='VALID'))
# mpool2
with tf.name_scope('mpool2'):
mpool2 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
# conv4
with tf.name_scope('conv4'):
weights = tf.Variable(
tf.truncated_normal([3, 3, conv3_uints, conv4_uints],
stddev=1.0 / math.sqrt(float(conv4_uints))),
name='weights')
biases = tf.Variable(tf.zeros(conv4_uints),
name='biases')
conv4 = tf.sigmoid(biases + tf.nn.conv2d(mpool2, weights,
strides=[1, 1, 1, 1],
padding='VALID'))
# mpool2
with tf.name_scope('mpool3'):
mpool3 = tf.nn.max_pool(conv4, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
# fc1
with tf.name_scope('fc1'):
#input_uints = int((BANDS_SIZE - conv1_kernel + 1) / 2)
print(mpool2.get_shape())
print(mpool3.get_shape())
x = mpool3.get_shape()[2].value
y = mpool3.get_shape()[3].value
z = mpool3.get_shape()[1].value
input_uints = x * y * z
weights = tf.Variable(
tf.truncated_normal([input_uints, fc1_uints],
stddev=1.0 / math.sqrt(float(input_uints))),
name='weights')
biases = tf.Variable(tf.zeros([fc1_uints]),
name='biases')
mpool_flat = tf.reshape(mpool3, [-1, input_uints])
#mpool_flat_drop = tf.nn.dropout(mpool_flat, keep_prob = 0.5)
#fc1 = tf.nn.relu(tf.matmul(mpool_flat_drop, weights) + biases)
fc1 = tf.sigmoid(tf.matmul(mpool_flat, weights) + biases)
# fc2
with tf.name_scope('fc2'):
weights = tf.Variable(
tf.truncated_normal([fc1_uints, fc2_uints],
stddev=1.0 / math.sqrt(float(fc1_uints))),
name='weights')
biases = tf.Variable(tf.zeros([fc2_uints]),
name='biases')
#fc1_drop = tf.nn.dropout(fc1, keep_prob = 0.5)
#fc2 = tf.nn.relu(tf.matmul(fc1_drop, weights) + biases)
fc2 = tf.sigmoid(tf.matmul(fc1, weights) + biases)
# softmax regression
with tf.name_scope('softmax_re'):
weights = tf.Variable(
tf.truncated_normal([fc2_uints, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(fc2_uints))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
softmax_re = tf.nn.softmax(tf.matmul(fc2, weights) + biases)
return softmax_re, conv4
# -*- coding: utf-8 -*-
import tensorflow as tf
sess = tf.Session()
x = tf.placeholder("float", shape=[None, 10], name="input")
W = tf.Variable(tf.truncated_normal([10, 5], stddev=0.05))
b = tf.Variable(tf.zeros([5]))
y = tf.nn.softmax(tf.matmul(x, W) + b, name="output")
sess.run(tf.initialize_all_variables())
#load data
mnist = input_data.read_data_sets('mnist_data', one_hot = True)
# size of batch
batch_size = 100
#all size
n_batch = mnist.train.num_examples // batch_size
#define two placeholder
x = tf.placeholder(tf.float32,[None, 784])
y = tf.placeholder(tf.float32,[None, 10])
lr = tf.Variable(0.001, dtype = tf.float32)
keep_prob = tf.placeholder(tf.float32)
#define nn
W1 = tf.Variable(tf.truncated_normal([784,500], stddev = 0.1))
b1 = tf.Variable(tf.zeros([500]) + 0.1)
L1 = tf.nn.tanh(tf.matmul(x,W1) + b1)
L1_drop = tf.nn.dropout(L1, keep_prob)
W2 = tf.Variable(tf.truncated_normal([500,300], stddev = 0.1))
b2 = tf.Variable(tf.zeros([300]) + 0.1)
L2 = tf.nn.tanh(tf.matmul(L1_drop,W2) + b2)
L2_drop = tf.nn.dropout(L2, keep_prob)
W3 = tf.Variable(tf.truncated_normal([300,100], stddev = 0.1))
b3 = tf.Variable(tf.zeros([100]) + 0.1)
L3 = tf.nn.tanh(tf.matmul(L2_drop,W3) + b3)
L3_drop = tf.nn.dropout(L3, keep_prob)
W4 = tf.Variable(tf.truncated_normal([100,10], stddev = 0.1))
def init_variable(shape):
return(tf.Variable(tf.truncated_normal(shape = shape)))
num_sampled = 64 # Number of negative examples to sample.
graph = tf.Graph()
with graph.as_default():
# Input data.
train_dataset = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Variables.
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
softmax_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
softmax_biases = tf.Variable(tf.zeros([vocabulary_size]), dtype=tf.float32)
# Model.
# Look up embeddings for inputs.
embed = tf.nn.embedding_lookup(embeddings, train_dataset)
# Compute the softmax loss, using a sample of the negative labels each time.
loss = tf.reduce_mean(tf.nn.sampled_softmax_loss(softmax_weights, softmax_biases, embed,train_labels, num_sampled, vocabulary_size))
# Optimizer.
# Note: The optimizer will optimize the softmax_weights AND the embeddings.
# This is because the embeddings are defined as a variable quantity and the
# optimizer's `minimize` method will by default modify all variable quantities
# that contribute to the tensor it is passed.
# See docs on `tf.train.Optimizer.minimize()` for more details.
optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
def main(argv=None): # pylint: disable=unused-argument
data_dir = './training/training/'
train_data_filename = data_dir + 'images/'
train_labels_filename = data_dir + 'groundtruth/'
# Extract it into numpy arrays.
train_data = extract_data(train_data_filename, TRAINING_SIZE)
train_labels = extract_labels(train_labels_filename, TRAINING_SIZE)
num_epochs = NUM_EPOCHS
c0 = 0
c1 = 0
for i in range(len(train_labels)):
if train_labels[i][0] == 1:
c0 = c0 + 1
else:
c1 = c1 + 1
print('Number of data points per class: c0 = ' + str(c0) + ' c1 = ' +
str(c1))
print('Balancing training data...')
min_c = min(c0, c1)
idx0 = [i for i, j in enumerate(train_labels) if j[0] == 1]
idx1 = [i for i, j in enumerate(train_labels) if j[1] == 1]
new_indices = idx0[0:min_c] + idx1[0:min_c]
print(len(new_indices))
print(train_data.shape)
train_data = train_data[new_indices, :, :, :]
train_labels = train_labels[new_indices]
train_size = train_labels.shape[0]
c0 = 0
c1 = 0
for i in range(len(train_labels)):
if train_labels[i][0] == 1:
c0 = c0 + 1
else:
c1 = c1 + 1
print('Number of data points per class: c0 = ' + str(c0) + ' c1 = ' +
str(c1))
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
train_data_node = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, IMG_PATCH_SIZE,
IMG_PATCH_SIZE, NUM_CHANNELS))
train_labels_node = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_LABELS))
train_all_data_node = tf.constant(train_data)
# The variables below hold all the trainable weights. They are passed an
# initial value which will be assigned when when we call:
# {tf.initialize_all_variables().run()}
conv1_weights = tf.Variable(
tf.truncated_normal(
[5, 5, NUM_CHANNELS, 32], # 5x5 filter, depth 32.
stddev=0.1,
seed=SEED))
conv1_biases = tf.Variable(tf.zeros([32]))
conv2_weights = tf.Variable(
tf.truncated_normal([5, 5, 32, 64], stddev=0.1, seed=SEED))
conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))
fc1_weights = tf.Variable( # fully connected, depth 512.
tf.truncated_normal(
[int(IMG_PATCH_SIZE / 4 * IMG_PATCH_SIZE / 4 * 64), 512],
stddev=0.1,
seed=SEED))
fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))
fc2_weights = tf.Variable(
tf.truncated_normal([512, NUM_LABELS], stddev=0.1, seed=SEED))
fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))
# Make an image summary for 4d tensor image with index idx
def get_image_summary(img, idx=0):
V = tf.slice(img, (0, 0, 0, idx), (1, -1, -1, 1))
img_w = img.get_shape().as_list()[1]
img_h = img.get_shape().as_list()[2]
min_value = tf.reduce_min(V)
V = V - min_value
max_value = tf.reduce_max(V)
V = V / (max_value * PIXEL_DEPTH)
V = tf.reshape(V, (img_w, img_h, 1))
V = tf.transpose(V, (2, 0, 1))
V = tf.reshape(V, (-1, img_w, img_h, 1))
return V
# Make an image summary for 3d tensor image with index idx
def get_image_summary_3d(img):
V = tf.slice(img, (0, 0, 0), (1, -1, -1))
img_w = img.get_shape().as_list()[1]
img_h = img.get_shape().as_list()[2]
V = tf.reshape(V, (img_w, img_h, 1))
V = tf.transpose(V, (2, 0, 1))
V = tf.reshape(V, (-1, img_w, img_h, 1))
return V
# Get prediction for given input image
def get_prediction(img):
data = numpy.asarray(img_crop(img, IMG_PATCH_SIZE, IMG_PATCH_SIZE))
data_node = tf.constant(data)
output = tf.nn.softmax(model(data_node))
output_prediction = s.run(output)
img_prediction = label_to_img(img.shape[0], img.shape[1],
IMG_PATCH_SIZE, IMG_PATCH_SIZE,
output_prediction)
return img_prediction
# Get a concatenation of the prediction and groundtruth for given input file
def get_prediction_with_groundtruth(filename, image_idx):
imageid = "satImage_%.3d" % image_idx
image_filename = filename + imageid + ".png"
img = mpimg.imread(image_filename)
img_prediction = get_prediction(img)
cimg = concatenate_images(img, img_prediction)
return cimg
# Get prediction overlaid on the original image for given input file
def get_prediction_with_overlay(filename, image_idx):
imageid = "satImage_%.3d" % image_idx
image_filename = filename + imageid + ".png"
img = mpimg.imread(image_filename)
img_prediction = get_prediction(img)
oimg = make_img_overlay(img, img_prediction)
return oimg
# We will replicate the model structure for the training subgraph, as well
# as the evaluation subgraphs, while sharing the trainable parameters.
def model(data, train=False):
"""The Model definition."""
# 2D convolution, with 'SAME' padding (i.e. the output feature map has
# the same size as the input). Note that {strides} is a 4D array whose
# shape matches the data layout: [image index, y, x, depth].
conv = tf.nn.conv2d(data,
conv1_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non-linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
# Max pooling. The kernel size spec {ksize} also follows the layout of
# the data. Here we have a pooling window of 2, and a stride of 2.
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv2 = tf.nn.conv2d(pool,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
pool2 = tf.nn.max_pool(relu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Uncomment these lines to check the size of each layer
# print 'data ' + str(data.get_shape())
# print 'conv ' + str(conv.get_shape())
# print 'relu ' + str(relu.get_shape())
# print 'pool ' + str(pool.get_shape())
# print 'pool2 ' + str(pool2.get_shape())
# Reshape the feature map cuboid into a 2D matrix to feed it to the
# fully connected layers.
pool_shape = pool2.get_shape().as_list()
reshape = tf.reshape(
pool2,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
# Fully connected layer. Note that the '+' operation automatically
# broadcasts the biases.
hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
# Add a 50% dropout during training only. Dropout also scales
# activations such that no rescaling is needed at evaluation time.
#if train:
# hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
out = tf.matmul(hidden, fc2_weights) + fc2_biases
if train == True:
summary_id = '_0'
s_data = get_image_summary(data)
filter_summary0 = tf.summary.image('summary_data' + summary_id,
s_data)
s_conv = get_image_summary(conv)
filter_summary2 = tf.summary.image('summary_conv' + summary_id,
s_conv)
s_pool = get_image_summary(pool)
filter_summary3 = tf.summary.image('summary_pool' + summary_id,
s_pool)
s_conv2 = get_image_summary(conv2)
filter_summary4 = tf.summary.image('summary_conv2' + summary_id,
s_conv2)
s_pool2 = get_image_summary(pool2)
filter_summary5 = tf.summary.image('summary_pool2' + summary_id,
s_pool2)
return out
# Training computation: logits + cross-entropy loss.
logits = model(train_data_node, True) # BATCH_SIZE*NUM_LABELS
# print 'logits = ' + str(logits.get_shape()) + ' train_labels_node = ' + str(train_labels_node.get_shape())
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=train_labels_node))
tf.summary.scalar('loss', loss)
all_params_node = [
conv1_weights, conv1_biases, conv2_weights, conv2_biases, fc1_weights,
fc1_biases, fc2_weights, fc2_biases
]
all_params_names = [
'conv1_weights', 'conv1_biases', 'conv2_weights', 'conv2_biases',
'fc1_weights', 'fc1_biases', 'fc2_weights', 'fc2_biases'
]
all_grads_node = tf.gradients(loss, all_params_node)
all_grad_norms_node = []
for i in range(0, len(all_grads_node)):
norm_grad_i = tf.global_norm([all_grads_node[i]])
all_grad_norms_node.append(norm_grad_i)
tf.summary.scalar(all_params_names[i], norm_grad_i)
# L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
# Add the regularization term to the loss.
loss += 5e-4 * regularizers
# Optimizer: set up a variable that's incremented once per batch and
# controls the learning rate decay.
batch = tf.Variable(0)
# Decay once per epoch, using an exponential schedule starting at 0.01.
learning_rate = tf.train.exponential_decay(
0.01, # Base learning rate.
batch * BATCH_SIZE, # Current index into the dataset.
train_size, # Decay step.
0.95, # Decay rate.
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# Use simple momentum for the optimization.
optimizer = tf.train.MomentumOptimizer(learning_rate,
0.0).minimize(loss,
global_step=batch)
# Predictions for the minibatch, validation set and test set.
train_prediction = tf.nn.softmax(logits)
# We'll compute them only once in a while by calling their {eval()} method.
train_all_prediction = tf.nn.softmax(model(train_all_data_node))
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a local session to run this computation.
with tf.Session() as s:
if RESTORE_MODEL:
# Restore variables from disk.
saver.restore(s, FLAGS.train_dir + "/model.ckpt")
print("Model restored.")
else:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph_def=s.graph_def)
print('Initialized!')
# Loop through training steps.
print('Total number of iterations = ' +
str(int(num_epochs * train_size / BATCH_SIZE)))
training_indices = range(train_size)
for iepoch in range(num_epochs):
# Permute training indices
perm_indices = numpy.random.permutation(training_indices)
for step in range(int(train_size / BATCH_SIZE)):
offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
batch_indices = perm_indices[offset:(offset + BATCH_SIZE)]
# Compute the offset of the current minibatch in the data.
# Note that we could use better randomization across epochs.
batch_data = train_data[batch_indices, :, :, :]
batch_labels = train_labels[batch_indices]
# This dictionary maps the batch data (as a numpy array) to the
# node in the graph is should be fed to.
feed_dict = {
train_data_node: batch_data,
train_labels_node: batch_labels
}
if step % RECORDING_STEP == 0:
summary_str, _, l, lr, predictions = s.run(
[
summary_op, optimizer, loss, learning_rate,
train_prediction
],
feed_dict=feed_dict)
#summary_str = s.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# print_predictions(predictions, batch_labels)
print('Epoch %.2f' %
(float(step) * BATCH_SIZE / train_size))
print('Minibatch loss: %.3f, learning rate: %.6f' %
(l, lr))
print('Minibatch error: %.1f%%' %
error_rate(predictions, batch_labels))
sys.stdout.flush()
else:
# Run the graph and fetch some of the nodes.
_, l, lr, predictions = s.run(
[optimizer, loss, learning_rate, train_prediction],
feed_dict=feed_dict)
# Save the variables to disk.
save_path = saver.save(s, FLAGS.train_dir + "/model.ckpt")
print("Model saved in file: %s" % save_path)
print("Running prediction on training set")
prediction_training_dir = "predictions_training/"
if not os.path.isdir(prediction_training_dir):
os.mkdir(prediction_training_dir)
for i in range(1, TRAINING_SIZE + 1):
pimg = get_prediction_with_groundtruth(train_data_filename, i)
Image.fromarray(pimg).save(prediction_training_dir +
"prediction_" + str(i) + ".png")
oimg = get_prediction_with_overlay(train_data_filename, i)
oimg.save(prediction_training_dir + "overlay_" + str(i) + ".png")
def cnn_weights(shape):
return tf.Variable(tf.truncated_normal(shape, stddev=0.1))
def main():
image_lists = create_image_lists(TEST_PERCENTAGE, VALIDATION_PERCENTAGE)
n_classes = len(image_lists.keys())
# 读取已经训练好的Inception-v3模型。谷歌训练好的模型包吃住了GraphDef ProtocolBuffer中，里面报错了每一个节点取值的计算方法及其变量的取值
# TensorFlow模型持久化的问题在第5章中有详细的计算
with gfile.FastGFile(os.path.join(MODEL_DIR, MODEL_FILE), 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
# 加载读取的Inception-v3模型，并返回数据输入所对应的张量以及计算瓶颈层结果对应的张量
bottleneck_tensor, jpeg_data_tensor = tf.import_graph_def(
graph_def, return_elements=[BOTTLENECK_TENSOR_NAME, JPEG_DATA_TENSOR_NAME])
# 定义新的神经网络输入，这个输入就是新的图片经过Inception-v3模型前向传播到达瓶颈层
# 时的取值。可以将这个过程类似的理解为一种特征提取
bottleneck_input = tf.placeholder(tf.float32, [None, BOTTLENECK_TENSOR_SIZE], name='BottleneckInputPlaceholder')
# 定义新的标准答案输入
ground_truth_input = tf.placeholder(tf.float32, [None, n_classes], name='GroundTruthInput')
# 定义一层全链接层来解决新的图片分类问题。因为训练好的Inception-v3模型已经将原始的图片抽象为了
# 更加容易分类的特征向量了，所有不需要再训练那么复杂的神经网络来完成新的分类任务
with tf.name_scope('final_training_ops'):
weights = tf.Variable(tf.truncated_normal([BOTTLENECK_TENSOR_SIZE, n_classes], stddev=0.001))
biases = tf.Variable(tf.zeros([n_classes]))
logits = tf.matmul(bottleneck_input, weights) + biases
final_tensor = tf.nn.softmax(logits)
# 定义交叉熵损失函数。
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=ground_truth_input)
cross_entropy_mean = tf.reduce_mean(cross_entropy)
train_step = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(cross_entropy_mean)
# 计算正确率。
with tf.name_scope('evaluation'):
correct_prediction = tf.equal(tf.argmax(final_tensor, 1), tf.argmax(ground_truth_input, 1))
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
# 训练过程。
for i in range(STEPS):
# 每次获取一个batch的训练数据
train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks(
sess, n_classes, image_lists, BATCH, 'training', jpeg_data_tensor, bottleneck_tensor)
sess.run(train_step,
feed_dict={bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth})
# 在验证数据上测试正确率
if i % 100 == 0 or i + 1 == STEPS:
validation_bottlenecks, validation_ground_truth = get_random_cached_bottlenecks(
sess, n_classes, image_lists, BATCH, 'validation', jpeg_data_tensor, bottleneck_tensor)
validation_accuracy = sess.run(evaluation_step, feed_dict={
bottleneck_input: validation_bottlenecks, ground_truth_input: validation_ground_truth})
print('Step %d: Validation accuracy on random sampled %d examples = %.1f%%' %
(i, BATCH, validation_accuracy * 100))
# 在最后的测试数据上测试正确率。
test_bottlenecks, test_ground_truth = get_test_bottlenecks(
sess, image_lists, n_classes, jpeg_data_tensor, bottleneck_tensor)
test_accuracy = sess.run(evaluation_step, feed_dict={
bottleneck_input: test_bottlenecks, ground_truth_input: test_ground_truth})
print('Final test accuracy = %.1f%%' % (test_accuracy * 100))
# fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))
# fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))
# fb = tf.Variable(tf.zeros([1, num_nodes]))
# 记忆门：输入，状态，偏置
# cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))
# cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))
# cb = tf.Variable(tf.zeros([1, num_nodes]))
# 输出门：输入，前一个输出，偏置
# ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))
# om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))
# ob = tf.Variable(tf.zeros([1, num_nodes]))
x_matrix = tf.Variable(
tf.truncated_normal([vocabulary_size, num_nodes * 4], -0.1, 0.1))
m_matrix = tf.Variable(
tf.truncated_normal([num_nodes, num_nodes * 4], -0.1, 0.1))
bias_matrix = tf.Variable(tf.zeros([1, num_nodes * 4]))
# 在展开之间保存hidden_value和state_value状态的变量。不可更新
saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]),
trainable=False)
saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]),
trainable=False)
# 分类器权重和偏置
# Classifier weights and biases.
w = tf.Variable(
tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))
b = tf.Variable(tf.zeros([vocabulary_size]))
def init_weight(self, dim_in, dim_out, name=None, stddev=1.0):
return tf.Variable(tf.truncated_normal([dim_in, dim_out], stddev=stddev/math.sqrt(float(dim_in))), name=name)
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Decay learning
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = 0.5
end_learning_rate = 0.05
decay_steps = 10000
learning_rate = tf.train.polynomial_decay(starter_learning_rate, global_step,
decay_steps, end_learning_rate,
power=1)
# Variables.
weights1 = tf.Variable(
tf.truncated_normal([image_size * image_size, hiddenlayer_num1],stddev=.12))
weights2 = tf.Variable(
tf.truncated_normal([hiddenlayer_num1, hiddenlayer_num2],stddev=.12))
weights3 = tf.Variable(
tf.truncated_normal([hiddenlayer_num2, hiddenlayer_num3],stddev=.12))
weights4 = tf.Variable(
tf.truncated_normal([hiddenlayer_num3, num_labels],stddev=.12))
biases1 = tf.Variable(tf.ones([hiddenlayer_num1]))
biases2 = tf.Variable(tf.ones([hiddenlayer_num2]))
biases3 = tf.Variable(tf.ones([hiddenlayer_num3]))
biases4 = tf.Variable(tf.ones([num_labels]))
# Training computation.
logits1 = tf.exp(-tf.square(tf.matmul(tf_train_dataset, weights1) + biases1))
logits2 = tf.exp(-tf.square(tf.matmul(logits1, weights2) + biases2))
def run_tf(num_nodes= 1024,batch_size = 128,num_steps = 10000,report_step=250, starter_learning_rate=0.1,regularization_lambda = 0.05, regularization=False):
input_size=X.shape[1]#dimension of each input vector
print('================num_nodes_hidden,batch_size:',num_nodes,batch_size)
print('================input_feature_size:',input_size)
print('================learning rate:',starter_learning_rate)
num_labels=2
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, input_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.the tf.Variable can be of any shape, or a tensor, so in this case it is the weight matrix of the dimension specified.this constructor gives the type and shpae of the Variable. After construction they are fixed but value can be changed using a assign method.actually the doc says you also can change the shape.
weights_1 = tf.Variable(
tf.truncated_normal([input_size, num_nodes]))
biases_1 = tf.Variable(tf.zeros([num_nodes]))
weights_2 = tf.Variable(
tf.truncated_normal([num_nodes, num_labels]))
biases_2 = tf.Variable(tf.zeros([num_labels]))
# Training computation.
relu_layer=tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)#notice the shape of tf_train_dataset and weights_1
logits = tf.matmul(relu_layer, weights_2) + biases_2
if regularization:
#L2 regularization
#regularization_lambda = 0.05
reg = regularization_lambda*tf.nn.l2_loss(weights_1) + regularization_lambda*tf.nn.l2_loss(weights_2)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels) + reg)
else:
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
#decay learning rate
global_step = tf.Variable(0, trainable=False)
#starter_learning_rate = 0.1
learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 10000, 0.96, staircase=True)
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2)
print ('=================num steps:',num_steps)
train_acc=[]
valid_acc=[]
train_F1=[]
valid_F1=[]
loss_log=[]
false_sgt_log=[]
false_nsgt_log=[]
with tf.Session(graph=graph) as session:
tf.initialize_all_variables().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
#print ("offset:",offset)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % report_step == 0):
#print("Minibatch loss at step %d: %f" % (step, l))
#print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
#print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
#print("Minibatch prec,recall,F1: %.3f %.3f %.3f" % precision_recall(predictions, batch_labels))
#print("validation prec,recall,F1: %.3f %.3f %.3f" % precision_recall(valid_prediction.eval(), valid_labels))
#get some error cases in training (not yet validation errors)
false_sgt,false_nsgt=get_error_cases(predictions,batch_labels)
false_sgt_batch=offset+np.array(false_sgt)
false_nsgt_batch=offset+np.array(false_nsgt)
false_sgt_log.append(false_sgt_batch)
false_nsgt_log.append(false_nsgt_batch)
loss_log.append(l)
train_acc.append(accuracy(predictions, batch_labels))
valid_acc.append(accuracy(valid_prediction.eval(), valid_labels))
train_F1.append(precision_recall(predictions, batch_labels))
valid_F1.append(precision_recall(valid_prediction.eval(), valid_labels))
#print("===================")
print ('final loss:', loss_log[-1])
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
print("Test precision, recall, F1: %.3f %.3f %.3f" % precision_recall(test_prediction.eval(), test_labels))
output_prob=False
if output_prob==True:
sgt_probs = [k for k in test_prediction.eval() if np.argmax(k) == 0]
for arr in sgt_probs:
print (arr[0],",")
plt.figure(0)
plot_tf(train_acc,valid_acc)
plt.title('accuracy')
plt.savefig('plots/accuracy.pdf')
#plot F1 scores
f1t=[i[2] for i in train_F1]
f1v=[i[2] for i in valid_F1]
plt.figure(1)
plot_tf(f1t,f1v)
plt.title('F1 score')
plt.savefig('plots/F1_score.pdf')
#plot precision
prect=[i[0] for i in train_F1]
precv=[i[0] for i in valid_F1]
plt.figure(2)
plot_tf(prect,precv)
plt.title('Precision')
plt.savefig('plots/precision.pdf')
# plot loss
plt.figure(3)
x=range(len(loss_log))
plt.plot(x,loss_log)
plt.ylim(0.4,1)
plt.title('loss function')
plt.savefig('plots/loss.pdf')
return train_F1,valid_F1,loss_log, false_sgt_log,false_nsgt_log
trainData = mnist.data[:60000].astype(float) / 255.
trainLabels = mnist.target[:60000]
testData = mnist.data[60000:].astype(float) / 255.
testLabels = mnist.target[60000:]
# Create the model
tf.reset_default_graph() # reset if we are rerunning code to avoid variable re-use
# Input layer
x = tf.placeholder(tf.float32, [None, 784])
# Hidden layer
#W1 = tf.get_variable('W1', [784, 100], initializer=tf.random_normal_initializer())
#W1 = tf.Variable(tf.random_normal([784, 100]))
W1 = tf.Variable(tf.truncated_normal([784, 100], stddev=1.0 / np.sqrt(784)))
b1 = tf.Variable(tf.zeros([100,]))
z1 = tf.matmul(x, W1) + b1
y1 = tf.nn.tanh(z1)
# Output layer
#W2 = tf.Variable(tf.random_normal([100, 10]))
W2 = tf.Variable(tf.truncated_normal([100, 10], stddev=1.0 / np.sqrt(100)))
b2 = tf.Variable(tf.zeros([10,]))
y2 = tf.matmul(y1, W2) + b2
# Define the output
y = y2
# Define loss and optimizer
# These will be inputs
## Input pixels, image with one channel (gray)
x = tf.placeholder("float", [None, 36, 36])
# Note that -1 is for reshaping
x_im = tf.reshape(x, [-1,36,36,1])
## Known labels
# None works during variable creation to be
# unspecified size
y_ = tf.placeholder("float", [None,5])
# Conv layer 1
num_filters = 4
winx = 5
winy = 5
W1 = tf.Variable(tf.truncated_normal(
[winx, winy, 1 , num_filters],
stddev=1./math.sqrt(winx*winy)))
b1 = tf.Variable(tf.constant(0.1,
shape=[num_filters]))
# 5x5 convolution, pad with zeros on edges
xw = tf.nn.conv2d(x_im, W1,
strides=[1, 1, 1, 1],
padding='SAME')
h1 = tf.nn.relu(xw + b1)
# 2x2 Max pooling, no padding on edges
p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='VALID')
# Need to flatten convolutional output for use in dense layer
p1_size = np.product(
[s.value for s in p1.get_shape()[1:]])
def defineGraph(self):
# with self.graph.as_default():
self.batchData()
#---- net variables
# eval_data = tf.place
with tf.name_scope('model'):
#This is where training samples and labels are fed to the graph.
#These placeholder nodes will be fed a batch of training data at each
#training step using the (feed_dict) argument to the Run() call below.
self.train_data_node = tf.placeholder(
data_type(),
shape=(BATCH_SIZE,IMG_H,IMG_W,IMG_CH),
name = 'train_data_node')
self.train_labels_node = tf.placeholder(data_type(),shape=(BATCH_SIZE,LABEL_W),
name = 'train_labels_node')
self.c1_w = tf.Variable(
tf.truncated_normal([8,8,IMG_CH,64],#5x5 filter depth 32.
stddev=0.1,
seed=SEED,dtype=data_type(),
name = 'c1_w'))
c1_b = tf.Variable(tf.zeros([64],dtype = data_type(),
name = 'c1_b'))
c2_w = tf.Variable(
tf.truncated_normal([4,4,64,32],
stddev=0.1,
seed=SEED,dtype = data_type(),
name = 'c2_w'))
c2_b = tf.Variable(tf.zeros([32],dtype = data_type(),
name = 'c2_b'))
fc1_w = tf.Variable(
tf.truncated_normal([IMG_H//4*IMG_W//4*32,1024],
stddev=0.1,
seed=SEED,dtype = data_type(),
name = 'fc1_w'))
fc1_b = tf.Variable(tf.constant(0.1,shape=[1024],dtype=data_type(),
name = 'fc1_b'))
fc2_w = tf.Variable(
tf.truncated_normal([1024,LABEL_W],
stddev=0.1,
seed=SEED,dtype = data_type(),
name = 'fc2_w'))
fc2_b = tf.Variable(tf.constant(0.1,shape=[LABEL_W],dtype=data_type(),
name = 'fc2_b'))
#-----------------------------
tf.summary.histogram('c1_w',self.c1_w)
#--------------------- end net variables
#
def model(data,train=False):
# shape matches the data layout:[image index,y,x,depth].
c1 = tf.nn.conv2d(data,
self.c1_w,
strides=[1,1,1,1],
padding = 'SAME')
# Bias and rectified linear non-linearity.
relu1 = tf.nn.relu(tf.nn.bias_add(c1,c1_b))
# Max pooling
pool1 = tf.nn.max_pool(relu1,
ksize=[1,2,2,1],
strides=[1,2,2,1],
padding='SAME')
c2 = tf.nn.conv2d(pool1,
c2_w,
strides=[1,1,1,1],
padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(c2,c2_b))
pool2 = tf.nn.max_pool(relu2,
ksize=[1,2,2,1],
strides=[1,2,2,1],
padding='SAME')
# Reshape the feature map cuboid into a 2D matrix to feed it to the fully connected layers.
poolShape = pool2.get_shape().as_list()
reshape = tf.reshape(
pool2,
[poolShape[0],poolShape[1]*poolShape[2]*poolShape[3]])
# Fully connected layer. Note that the '+' operation automatically broadcasts the biases.
fc1 = tf.nn.tanh(tf.matmul(reshape,fc1_w) + fc1_b)
# Add a 50% dropout during training training only.
# Dropout also scales activations such that no rescaling is needed at evaluation time
if train:
fc1 = tf.nn.dropout(fc1,0.5,seed=SEED)
return tf.nn.sigmoid(tf.matmul(fc1,fc2_w) + fc2_b)
# return (tf.matmul(fc1,fc2_w) + fc2_b)
#--------------------end model
# Training computation: logits + cross-entropy loss
logits = model(self.train_data_node,self.train)
self.loss1 = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,labels=self.train_labels_node))
# L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_w) + tf.nn.l2_loss(fc1_b) + tf.nn.l2_loss(fc2_w) + tf.nn.l2_loss(fc2_b))
# Add the regularization term to the loss.
#
# if tf.is_nan(regularizers) is not None:
# self.loss = self.loss1*(1 + tf.nn.tanh(1e-2*regularizers))
# else:
# self.loss = self.loss1
# self.loss = self.loss1 + 2e-7*regularizers
## regularizers = tf.nn.tanh(regularizers)
## self.loss = self.loss1 + regularizers
self.loss = self.loss1
tf.summary.histogram('loss1',self.loss1)
tf.summary.histogram('loss',self.loss)
# Optimizer: set up a variable that's incremented once per batch and controls the learning rate decay.
batch = tf.Variable(0,dtype=data_type())
# Decay once per epoch, using an exponential schedule starting at 0.01
self.learningRate = tf.train.exponential_decay(
0.05, # Base learning rate.
batch * self.batchSize, # Current index into the dataset
DECAY_STEP, # Decay step.
0.99, # Decay rate.
staircase=True)
# learningRate = 0.1
self.optimizer = tf.train.MomentumOptimizer(self.learningRate,0.9).minimize(self.loss,global_step=batch)
# Predictions for the current training minibatch
# self.trainPrediction = tf.nn.sigmoid(logits)
self.trainPrediction = (logits)
def fc_layer(input, size_in, size_out, name="fc"):
with tf.name_scope(name):
w = tf.Variable(tf.truncated_normal([size_in, size_out], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_out]), name="B")
act = tf.nn.relu(tf.matmul(input, w) + b)
return act
def weight_variable_noise(shape):
return tf.Variable(0.1 * tf.truncated_normal(shape=shape, stddev=0.1))
tf.summary.scalar('max',tf.reduce_max(var))
tf.summary.scalar('min',tf.reduce_min(var))
tf.summary.histogram('histogram',var)
#定义命名空间
with tf.name_scope('input'):
#定义placeholder
x=tf.placeholder(tf.float32,[None,784],name='x-input')
y=tf.placeholder(tf.float32,[None,10],name='y-input')
keep_prob=tf.placeholder(tf.float32)
lr=tf.Variable(0.001,dtype=tf.float32)
#定义神经网络输入层
with tf.name_scope('L1_layer'):
with tf.name_scope('W_L1'):
W_L1=tf.Variable(tf.truncated_normal([784,100],stddev=0.1))
variable_summaries(W_L1)
with tf.name_scope('b_L1'):
b_L1=tf.Variable(tf.zeros([100])+0.1)
variable_summaries(b_L1)
with tf.name_scope('wx_plus_b_L1'):
wx_plus_b_L1=tf.matmul(x,W_L1)+b_L1
with tf.name_scope('tanh'):
L1=tf.nn.tanh(wx_plus_b_L1)
with tf.name_scope('dropout'):
L1_dropout=tf.nn.dropout(L1,keep_prob)
#定义神经网络隐藏层
with tf.name_scope('L2_layer'):
with tf.name_scope('W_L2'):
W_L2=tf.Variable(tf.truncated_normal([100,50],stddev=0.1))
def weight(shape, stddev=0.1, mean=0):
initial = tf.truncated_normal(shape=shape, mean=mean, stddev=stddev)
return tf.Variable(initial)
def weight_variable(name, shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.get_variable(name, initializer=initial)
def _build_model(self):
embedding_size = 128
filter_sizes = [3, 4, 5]
num_filters = 128
with tf.variable_scope('embedding'):
embeddings = tf.Variable(
tf.random_uniform([self.vocab_size, embedding_size], -1.0,
1.0))
#对于每段文字 转换为词向量的矩阵表示
embeddings_char = tf.nn.embedding_lookup(embeddings, self.input_x)
embeddings_char_expanded = tf.expand_dims(embeddings_char, -1)
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope('conv-maxpool-%s' % filter_size):
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1),
name='W')
b = tf.Variable(tf.constant(0.1, shape=[num_filters]),
name='b')
conv = tf.nn.conv2d(embeddings_char_expanded,
W,
strides=[1, 1, 1, 1],
padding='VALID',
name='conv')
#Relu激活函数，将非最大值置为0
h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')
# 池化操作 取出卷积后的最大值
pooled = tf.nn.max_pool(
h,
ksize=[1, self.seq_len - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name='pool')
pooled_outputs.append(pooled)
num_filters_total = num_filters * len(filter_sizes)
h_pool = tf.concat(3, pooled_outputs)
h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])
h_dropout = tf.nn.dropout(h_pool_flat, self.keep_prob)
l2_loss = tf.constant(0.0)
with tf.name_scope('output'):
W = tf.get_variable(
"W",
shape=[num_filters_total, self.num_classes],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.constant(0.1, shape=[self.num_classes]),
name='b')
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
logits = tf.nn.xw_plus_b(h_dropout, W, b, name='scores')
#返回最大值的序号
self.predictions = tf.argmax(logits, 1, name='predictions')
#利用softmax计算损失函数
with tf.name_scope('loss'):
losses = tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=self.input_y)
self.loss = tf.reduce_mean(losses) + self.l2_reg_lambda * l2_loss
with tf.name_scope('accuracy'):
correct_predictions = tf.equal(self.predictions,
tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions,
'float'),
name='accuracy')
self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(
self.loss)
self.saver = tf.train.Saver(tf.global_variables())
print('Built model!')
with tf.device('/cpu:0'):
# 模型参数
learning_rate = 1e-4
training_iters = 200
batch_size = 200
display_step = 5
n_classes = 2
n_fc1 = 4096
n_fc2 = 2048
# 构建模型
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.int32, [None, n_classes])
W_conv = {
'conv1': tf.Variable(tf.truncated_normal([11, 11, 3, 96], stddev=0.0001)),
'conv2': tf.Variable(tf.truncated_normal([5, 5, 96, 256], stddev=0.01)),
'conv3': tf.Variable(tf.truncated_normal([3, 3, 256, 384], stddev=0.01)),
'conv4': tf.Variable(tf.truncated_normal([3, 3, 384, 384], stddev=0.01)),
'conv5': tf.Variable(tf.truncated_normal([3, 3, 384, 256], stddev=0.01)),
'fc1': tf.Variable(tf.truncated_normal([13 * 13 * 256, n_fc1], stddev=0.1)),
'fc2': tf.Variable(tf.truncated_normal([n_fc1, n_fc2], stddev=0.1)),
'fc3': tf.Variable(tf.truncated_normal([n_fc2, n_classes], stddev=0.1))
}
b_conv = {
'conv1': tf.Variable(tf.constant(0.0, dtype=tf.float32, shape=[96])),
'conv2': tf.Variable(tf.constant(0.1, dtype=tf.float32, shape=[256])),
'conv3': tf.Variable(tf.constant(0.1, dtype=tf.float32, shape=[384])),
'conv4': tf.Variable(tf.constant(0.1, dtype=tf.float32, shape=[384])),
'conv5': tf.Variable(tf.constant(0.1, dtype=tf.float32, shape=[256])),
'fc1': tf.Variable(tf.constant(0.1, dtype=tf.float32, shape=[n_fc1])),
'''
tf.truncated_normal
truncated_normal(
shape,
mean=0.0,
stddev=1.0,
dtype=tf.float32,
seed=None,
name=None
)
产生截断正态分布随机数，取值范围为 [ mean - 2 * stddev, mean + 2 * stddev ]。
参数列表：
参数名 必选 类型 说明
shape 是 1 维整形张量或 array 输出张量的维度
mean 否 0 维张量或数值 均值
stddev 否 0 维张量或数值 标准差
dtype 否 dtype 输出类型
seed 否 数值 随机种子，若 seed 赋值，每次产生相同随机数
name 否 string 运算名称
'''
import tensorflow as tf
initial = tf.truncated_normal(shape=[3,3], mean=0, stddev=1)
print(tf.Session().run(initial))
def train_province():
global iterations_P
global time_begin
#if __name__ == '__main__' and sys.argv[1] == 'train':
# 第一次遍历图片目录是为了获取图片总数
input_count = 0
for i in range(0, NUM_CLASSES_P):
dir = './train_images/training-set/chinese-characters/%s/' % i # 这里可以改成你自己的图片目录，i为分类标签
for rt, dirs, files in os.walk(dir):
for filename in files:
input_count += 1
# 定义对应维数和各维长度的数组
input_images = np.array([[0] * SIZE for i in range(input_count)])
input_labels = np.array([[0] * NUM_CLASSES_P for i in range(input_count)])
# 第二次遍历图片目录是为了生成图片数据和标签
index = 0
for i in range(0, NUM_CLASSES_P):
dir = './train_images/training-set/chinese-characters/%s/' % i # 这里可以改成你自己的图片目录，i为分类标签
for rt, dirs, files in os.walk(dir):
for filename in files:
filename = dir + filename
img = Image.open(filename)
width = img.size[0]
height = img.size[1]
for h in range(0, height):
for w in range(0, width):
# 通过这样的处理，使数字的线条变细，有利于提高识别准确率
if img.getpixel((w, h)) > 230:
input_images[index][w + h * width] = 0
else:
input_images[index][w + h * width] = 1
input_labels[index][i] = 1
index += 1
# 第一次遍历图片目录是为了获取图片总数
val_count = 0
for i in range(0, NUM_CLASSES_P):
dir = './train_images/validation-set/chinese-characters/%s/' % i # 这里可以改成你自己的图片目录，i为分类标签
for rt, dirs, files in os.walk(dir):
for filename in files:
val_count += 1
# 定义对应维数和各维长度的数组
val_images = np.array([[0] * SIZE for i in range(val_count)])
val_labels = np.array([[0] * NUM_CLASSES_P for i in range(val_count)])
# 第二次遍历图片目录是为了生成图片数据和标签
index = 0
for i in range(0, NUM_CLASSES_P):
dir = './train_images/validation-set/chinese-characters/%s/' % i # 这里可以改成你自己的图片目录，i为分类标签
for rt, dirs, files in os.walk(dir):
for filename in files:
filename = dir + filename
img = Image.open(filename)
width = img.size[0]
height = img.size[1]
for h in range(0, height):
for w in range(0, width):
# 通过这样的处理，使数字的线条变细，有利于提高识别准确率
if img.getpixel((w, h)) > 230:
val_images[index][w + h * width] = 0
else:
val_images[index][w + h * width] = 1
val_labels[index][i] = 1
index += 1
with tf.Session() as sess:
# 第一个卷积层
W_conv1 = tf.Variable(tf.truncated_normal([8, 8, 1, 16], stddev=0.1),
name="W_conv1")
b_conv1 = tf.Variable(tf.constant(0.1, shape=[16]), name="b_conv1")
conv_strides = [1, 1, 1, 1]
kernel_size = [1, 2, 2, 1]
pool_strides = [1, 2, 2, 1]
L1_pool = conv_layer(x_image,
W_conv1,
b_conv1,
conv_strides,
kernel_size,
pool_strides,
padding='SAME')
# 第二个卷积层
W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 16, 32], stddev=0.1),
name="W_conv2")
b_conv2 = tf.Variable(tf.constant(0.1, shape=[32]), name="b_conv2")
conv_strides = [1, 1, 1, 1]
kernel_size = [1, 1, 1, 1]
pool_strides = [1, 1, 1, 1]
L2_pool = conv_layer(L1_pool,
W_conv2,
b_conv2,
conv_strides,
kernel_size,
pool_strides,
padding='SAME')
# 全连接层
W_fc1 = tf.Variable(tf.truncated_normal([16 * 20 * 32, 512],
stddev=0.1),
name="W_fc1")
b_fc1 = tf.Variable(tf.constant(0.1, shape=[512]), name="b_fc1")
h_pool2_flat = tf.reshape(L2_pool, [-1, 16 * 20 * 32])
h_fc1 = full_connect(h_pool2_flat, W_fc1, b_fc1)
# dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# readout层
W_fc2 = tf.Variable(tf.truncated_normal([512, NUM_CLASSES_P],
stddev=0.1),
name="W_fc2")
b_fc2 = tf.Variable(tf.constant(0.1, shape=[NUM_CLASSES_P]),
name="b_fc2")
# 定义优化器和训练op
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_P, logits=y_conv))
train_step = tf.train.AdamOptimizer((1e-4)).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_P, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# 初始化saver
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
time_elapsed = time.time() - time_begin
print("读取图片文件耗费时间：%d秒" % time_elapsed)
time_begin = time.time()
print("一共读取了 %s 个训练图像， %s 个标签" % (input_count, input_count))
# 设置每次训练op的输入个数和迭代次数，这里为了支持任意图片总数，定义了一个余数remainder，譬如，如果每次训练op的输入个数为60，图片总数为150张，则前面两次各输入60张，最后一次输入30张（余数30）
batch_size = 60
iterations_P = iterations_P
batches_count = int(input_count / batch_size)
remainder = input_count % batch_size
print("训练数据集分成 %s 批, 前面每批 %s 个数据，最后一批 %s 个数据" %
(batches_count + 1, batch_size, remainder))
# 执行训练迭代
for it in range(iterations_P):
# 这里的关键是要把输入数组转为np.array
for n in range(batches_count):
train_step.run(
feed_dict={
x: input_images[n * batch_size:(n + 1) * batch_size],
y_P: input_labels[n * batch_size:(n + 1) * batch_size],
keep_prob: 0.5
})
if remainder > 0:
start_index = batches_count * batch_size
train_step.run(
feed_dict={
x: input_images[start_index:input_count - 1],
y_P: input_labels[start_index:input_count - 1],
keep_prob: 0.5
})
# 每完成五次迭代，判断准确度是否已达到100%，达到则退出迭代循环
iterate_accuracy = 0
if it % 5 == 0:
iterate_accuracy = accuracy.eval(feed_dict={
x: val_images,
y_P: val_labels,
keep_prob: 1.0
})
print('第 %d 次训练迭代: 准确率 %0.5f%%' % (it, iterate_accuracy * 100))
if iterate_accuracy >= 0.995 and it >= 150:
break
print('完成训练!')
time_elapsed = time.time() - time_begin
print("训练耗费时间：%d秒" % time_elapsed)
time_begin = time.time()
# 保存训练结果
if not os.path.exists(SAVER_DIR_P):
print('不存在训练数据保存目录，现在创建保存目录')
os.makedirs(SAVER_DIR_P)
saver_path = saver.save(sess, "%smodel.ckpt" % (SAVER_DIR_P))
