def create_conv(prev, filter_size, nb):
w_filters = tf.Variable(
tf.truncated_normal(shape=(filter_size, filter_size,
int(prev.get_shape()[-1]), nb)))
b_filters = tf.Variable(tf.zeros(shape=(nb)))
conv = tf.nn.conv2d(prev, w_filters, strides=[1, 1, 1, 1],
padding="SAME") + b_filters
conv = tf.nn.relu(conv)
conv = tf.nn.max_pool(conv,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="SAME")
return conv
def init_weights(shape):
'''
Initialize weight of a layer using a normal distribution
Args:
- shape: shape of the weights
Returns:
- W (tf.Variable): initialized weights of shape `input_shape`
'''
init_weight = tf.truncated_normal(shape=shape, stddev=0.1)
W = tf.Variable(init_weight)
return W
def buildGraph(loss="MSE", beta1=0.9, beta2=0.999, epsilon=1e-08):
W = tf.Variable(tf.truncated_normal(shape=[28 * 28, 1], stddev=0.5, name="weights"))
b = tf.Variable(0.0, name="biases")
x = tf.placeholder(tf.float32, shape=[None, 28 * 28])
y = tf.placeholder(tf.float32, shape=[None, 1])
validData = tf.placeholder(tf.float32, shape=[None, 28 * 28])
validTarget = tf.placeholder(tf.float32, shape=[None, 1])
testData = tf.placeholder(tf.float32, shape=[None, 28 * 28])
testTarget = tf.placeholder(tf.float32, shape=[None, 1])
reg = tf.placeholder(tf.float32)
tf.set_random_seed(421)
if loss == "MSE":
y_pred = tf.matmul(x, W) + b
valid_pred = tf.matmul(validData, W) + b
test_pred = tf.matmul(testData, W) + b
train_loss = tf.losses.mean_squared_error(labels=y, predictions=y_pred) + \
reg * tf.nn.l2_loss(W)
optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=beta1, \
beta2=beta2, epsilon=epsilon)
train = optimizer.minimize(loss=train_loss)
valid_loss = tf.losses.mean_squared_error(labels=validTarget, \
redictions=valid_pred) + reg * tf.nn.l2_loss(W)
test_loss = tf.losses.mean_squared_error(labels=testTarget, \
predictions=test_pred) + reg * tf.nn.l2_loss(W)
elif loss == "CE":
train_logits = tf.matmul(x, W) + b
valid_logits = tf.matmul(validData, W) + b
test_logits = tf.matmul(testData, W) + b
train_loss = tf.losses.sigmoid_cross_entropy(y, train_logits) + \
reg * tf.nn.l2_loss(W)
optimizer = tf.train.AdamOptimizer(learning_rate=0.001, \
beta1=beta1, beta2=beta2, epsilon=epsilon)
train = optimizer.minimize(loss=train_loss)
valid_loss = tf.losses.sigmoid_cross_entropy(validTarget, valid_logits) + \
reg * tf.nn.l2_loss(W)
test_loss = tf.losses.sigmoid_cross_entropy(testTarget, test_logits) + \
reg * tf.nn.l2_loss(W)
y_pred = tf.sigmoid(train_logits)
valid_pred = tf.sigmoid(valid_logits)
test_pred = tf.sigmoid(test_logits)
return W, b, x, y_pred, y, validData, validTarget, testData, testTarget, reg, \
train_loss, valid_loss, test_loss, train, valid_pred, test_pred
def upsample_and_concat(x1, x2, output_channels, in_channels):
pool_size = 2
deconv_filter = tf.Variable(
tf.truncated_normal(
[pool_size, pool_size, output_channels, in_channels], stddev=0.02))
deconv = tf.nn.conv2d_transpose(x1,
deconv_filter,
tf.shape(x2),
strides=[1, pool_size, pool_size, 1])
deconv_output = tf.concat([deconv, x2], 3)
deconv_output.set_shape([None, None, None, output_channels * 2])
return deconv_output
def kaiming(shape, dtype, partition_info=None):
"""Kaiming initialization as described in https://arxiv.org/pdf/1502.01852.pdf
Args
shape: dimensions of the tf array to initialize
dtype: data type of the array
partition_info: (Optional) info about how the variable is partitioned.
See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/init_ops.py#L26
Needed to be used as an initializer.
Returns
Tensorflow array with initial weights
"""
return (tf.truncated_normal(shape, dtype=dtype) *
tf.sqrt(2 / float(shape[0])))
def __init__(self, vehicles_train_path, non_vehicles_train_path, vehicles_test_path, non_vehicles_test_path,
image_shape: tuple, batch_size: int, train_classes_ratio: Fraction,
train_rate: float, train_num_iterate):
self.__batch_size__ = batch_size
self.__train_classes_ratio__ = train_classes_ratio
self.__images_cntrl__ = ImagesController(vehicles_train_path, non_vehicles_train_path,
vehicles_test_path, non_vehicles_test_path, self.__batch_size__,
self.__train_classes_ratio__)
self.__image_shape__ = image_shape
self.__train_num_iterate__ = train_num_iterate
self.__train_rate__ = train_rate
self.__num_inputs__ = image_shape[0] * image_shape[1]
self.__input__ = tf.placeholder(tf.float32, [None, self.__num_inputs__])
self.__weights__ = tf.Variable(tf.truncated_normal(shape=[self.__num_inputs__, 1], stddev=0.1))
self.__bias__ = tf.Variable(tf.truncated_normal(shape=[1], stddev=0.1))
self.__output__ = tf.nn.sigmoid(tf.matmul(self.__input__, self.__weights__) + self.__bias__)
self.__real__ = tf.placeholder(tf.float32, [None, 1])
self.__loss__ = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=self.__real__,
logits=self.__output__))
self.__update__ = tf.train.GradientDescentOptimizer(self.__train_rate__).minimize(self.__loss__)
self.__sess__ = tf.Session()
self.__initialized__ = False
self.__closed__ = False
def _init_conv_layers(self):
self.__conv_weights__ = []
self.__conv_biases__ = []
self.__conv_outputs__ = []
self.__conv_activations__ = []
self.__conv_pools__ = []
kernel_r = self.__conv_kernel_dimensions__[
0] # First dimension of each kernel. Numbers of rows.
kernel_c = self.__conv_kernel_dimensions__[
0] # Second dimension of each kernel. Numbers of columns.
for (i, num) in enumerate(self.__conv_layers_num_kernels__):
if i == 0:
prev_dim = 1
prev_pool = self.__conv_input_batch__
else:
prev_dim = self.__conv_layers_num_kernels__[i - 1]
prev_pool = self.__conv_pools__[i - 1]
self.__conv_weights__.append(
tf.Variable(
tf.truncated_normal([kernel_r, kernel_c, prev_dim, num],
stddev=0.1)))
# self.__conv_biases__.append(tf.Variable(tf.constant(0.1, shape=[num])))
self.__conv_biases__.append(
tf.Variable(tf.truncated_normal(shape=[num], stddev=0.1)))
self.__conv_outputs__.append(
tf.nn.conv2d(prev_pool,
self.__conv_weights__[i],
strides=[1, 1, 1, 1],
padding=ConvNetwork.CONST_PADDING) +
self.__conv_biases__[i])
self.__conv_activations__.append(
tf.nn.relu(self.__conv_outputs__[i]))
self.__conv_pools__.append(
tf.nn.max_pool(self.__conv_activations__[i],
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding=ConvNetwork.CONST_PADDING))
def body(self, features):
"""TextCNN main model_fn.
Args:
features: Map of features to the model. Should contain the following:
"inputs": Text inputs.
[batch_size, input_length, 1, hidden_dim].
"targets": Target encoder outputs.
[batch_size, 1, 1, hidden_dim]
Returns:
Final encoder representation. [batch_size, 1, 1, hidden_dim]
"""
hparams = self._hparams
inputs = features["inputs"]
xshape = common_layers.shape_list(inputs)
vocab_size = xshape[3]
inputs = tf.reshape(inputs, [xshape[0], xshape[1], xshape[3], xshape[2]])
pooled_outputs = []
for _, d_ff in enumerate(hparams.d_ffs):
with tf.name_scope("conv-maxpool-%s" % d_ff):
filter_shape = [d_ff, vocab_size, 1, hparams.num_filters]
filter_var = tf.Variable(
tf.truncated_normal(filter_shape, stddev=0.1), name="W")
filter_bias = tf.Variable(
tf.constant(0.1, shape=[hparams.num_filters]), name="b")
conv = tf.nn.conv2d(
inputs,
filter_var,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
conv_outputs = tf.nn.relu(
tf.nn.bias_add(conv, filter_bias), name="relu")
pooled = tf.math.reduce_max(
conv_outputs, axis=1, keepdims=True, name="max")
pooled_outputs.append(pooled)
num_filters_total = hparams.num_filters * len(hparams.d_ffs)
h_pool = tf.concat(pooled_outputs, 3)
h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])
# Add dropout
output = tf.nn.dropout(h_pool_flat, 1 - hparams.output_dropout)
output = tf.reshape(output, [-1, 1, 1, num_filters_total])
return output
def conv2d(input):
# Filter (weights and bias)
# The shape of the filter weight is (height, width, input_depth, output_depth)
# The shape of the filter bias is (output_depth,)
# TODO: Define the filter weights `F_W` and filter bias `F_b`.
# NOTE: Remember to wrap them in `tf.Variable`, they are trainable parameters after all.
F_W = tf.Variable(tf.truncated_normal((2, 2, 1, 3)))
F_b = tf.Variable(tf.zeros(3))
# TODO: Set the stride for each dimension (batch_size, height, width, depth)
strides = [1, 2, 2, 1]
# TODO: set the padding, either 'VALID' or 'SAME'.
padding = 'VALID'
# https://www.tensorflow.org/versions/r0.11/api_docs/python/nn.html#conv2d
# `tf.nn.conv2d` does not include the bias computation so we have to add it ourselves after.
return tf.nn.conv2d(input, F_W, strides, padding) + F_b
def fc_layer(self,idx,inputs,hiddens,flat = False,linear = False):
input_shape = inputs.get_shape().as_list()
if flat:
dim = input_shape[1]*input_shape[2]*input_shape[3]
inputs_transposed = tf.transpose(inputs,(0,3,1,2))
inputs_processed = tf.reshape(inputs_transposed, [-1,dim])
else:
dim = input_shape[1]
inputs_processed = inputs
weight = tf.Variable(tf.truncated_normal([dim,hiddens], stddev=0.1))
biases = tf.Variable(tf.constant(0.1, shape=[hiddens]))
print ('Layer %d : Type = Full, Hidden = %d, Input dimension = %d, Flat = %d, Activation = %d' % (idx,hiddens,int(dim),int(flat),1-int(linear)) )
if linear : return tf.add(tf.matmul(inputs_processed,weight),biases,name=str(idx)+'_fc')
ip = tf.add(tf.matmul(inputs_processed,weight),biases)
return tf.maximum(self.alpha*ip,ip,name=str(idx)+'_fc')
def model(batch_x):
#weights of neural network
weights = {
'w1':
tf.Variable(tf.truncated_normal([n_input, n_hidden1], stddev=0.1),
name="w1"),
'w2':
tf.Variable(tf.truncated_normal([n_hidden1, n_hidden2], stddev=0.1),
name="w2"),
'w3':
tf.Variable(tf.truncated_normal([n_hidden2, n_hidden3], stddev=0.1),
name="w3"),
'out':
tf.Variable(tf.truncated_normal([n_hidden3, n_class], stddev=0.1),
name="wout"),
}
#biases of neural network
biases = {
'b1': tf.Variable(tf.zeros([n_hidden1]), name="b1"),
'b2': tf.Variable(tf.zeros([n_hidden2]), name="b2"),
'b3': tf.Variable(tf.zeros([n_hidden3]), name="b3"),
'out': tf.Variable(tf.zeros([n_class]), name="bout")
}
#neural netork operations
#layer1
layer_1 = tf.add(tf.matmul(batch_x, weights['w1']), biases['b1'])
layer_1 = tf.maximum(0.0, layer_1)
#layer2
layer_2 = tf.add(tf.matmul(layer_1, weights['w2']), biases['b2'])
layer_2 = tf.maximum(0.0, layer_2)
#layer3
layer_3 = tf.add(tf.matmul(layer_2, weights['w3']), biases['b3'])
layer_3 = tf.maximum(0.0, layer_3)
#output
output_layer = tf.matmul(layer_3, weights['out']) + biases['out']
return output_layer
def phi_R(B):
'''
The phi_R function that predict the effect of each interaction by applying f_R to each column of B
'''
h_size = 50
B_trans = tf.transpose(B, [0, 2, 1])
B_trans = tf.reshape(B_trans, [-1, (Ds + Dr)])
w1 = tf.Variable(tf.random.truncated_normal([(Ds + Dr), h_size],
stddev=0.1),
name="r_w1",
dtype=tf.float32)
b1 = tf.Variable(tf.zeros([h_size]), name="r_b1", dtype=tf.float32)
h1 = tf.nn.relu(tf.matmul(B_trans, w1) + b1)
w2 = tf.Variable(tf.random.truncated_normal([h_size, h_size], stddev=0.1),
name="r_w2",
dtype=tf.float32)
b2 = tf.Variable(tf.zeros([h_size]), name="r_b2", dtype=tf.float32)
h2 = tf.nn.relu(tf.matmul(h1, w2) + b2)
w3 = tf.Variable(tf.random.truncated_normal([h_size, h_size], stddev=0.1),
name="r_w3",
dtype=tf.float32)
b3 = tf.Variable(tf.zeros([h_size]), name="r_b3", dtype=tf.float32)
h3 = tf.nn.relu(tf.matmul(h2, w3) + b3)
w4 = tf.Variable(tf.truncated_normal([h_size, h_size], stddev=0.1),
name="r_w4",
dtype=tf.float32)
b4 = tf.Variable(tf.zeros([h_size]), name="r_b4", dtype=tf.float32)
h4 = tf.nn.relu(tf.matmul(h3, w4) + b4)
w5 = tf.Variable(tf.truncated_normal([h_size, De], stddev=0.1),
name="r_w5",
dtype=tf.float32)
b5 = tf.Variable(tf.zeros([De]), name="r_b5", dtype=tf.float32)
h5 = tf.matmul(h4, w5) + b5
h5_trans = tf.reshape(h5, [-1, Nr, De])
h5_trans = tf.transpose(h5_trans, [0, 2, 1])
return h5_trans
def linear_network(x_in: int,
in_dim: int,
out_dim: int,
mu_in=mu,
sigma_in=sigma):
"""
Create a linear network layer with the input parameters provided.
"""
W = tf.Variable(
tf.truncated_normal(shape=(in_dim, out_dim),
mean=mu_in,
stddev=sigma_in))
B = tf.Variable(tf.zeros(shape=(1, out_dim)))
y_out = tf.matmul(x_in, W) + B
return y_out
def build_model(num_features, num_classes):
# Number of features
n = num_features
#Number of classes.
K = num_classes
# There are n columns in the feature matrix
X = tf.placeholder(tf.float32, [None, n])
# Label is mxK matrix.
Y = tf.placeholder(tf.float32, [None, K])
# Weights. nxK matrix
W = tf.Variable(tf.truncated_normal([n, K], stddev=0.001))
# Bias. One for each class.
b = tf.Variable(tf.truncated_normal([K], stddev=0.001))
# Hypothesis uses softmax.
#It is a mxK matrix
logits = tf.matmul(X, W) + b
Y_hat = tf.nn.softmax(logits)
# Cost function for softmax hypotheses function
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=logits))
# Gradient Descent Optimizer
optimizer = tf.train.GradientDescentOptimizer(0.001).minimize(cost)
#Index of the highest predicted value is taken as
#the final decision.
correct_prediction = tf.equal(tf.argmax(Y_hat, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return optimizer, X, Y, Y_hat, cost, accuracy
def inception_v3(nlabels, images):
batch_norm_params = {
"is_training": False, "trainable": True, "decay": 0.9997,
"epsilon": 0.001,
"variables_collections": {
"beta": None,
"gamma": None,
"moving_mean": ["moving_vars"],
"moving_variance": ["moving_vars"],
}
}
weight_decay = 0.00004
stddev = 0.1
weights_regularizer = tf_slim.l2_regularizer(weight_decay)
args_for_scope = (
dict(list_ops_or_scope=[tf_slim.layers.conv2d, tf_slim.layers.fully_connected],
weights_regularizer=weights_regularizer, trainable=True),
dict(list_ops_or_scope=[tf_slim.layers.conv2d],
weights_initializer=tf1.truncated_normal_initializer(stddev=stddev),
activation_fn=tf1.nn.relu,
normalizer_fn=tf_slim.layers.batch_norm,
normalizer_params=batch_norm_params),
)
with tf1.variable_scope("InceptionV3", "InceptionV3", [images]) as scope, \
tf_slim.arg_scope(**args_for_scope[0]), \
tf_slim.arg_scope(**args_for_scope[1]):
net, end_points = inception_v3_base(images, scope=scope)
with tf1.variable_scope("logits"):
shape = net.get_shape()
net = tf_slim.layers.avg_pool2d(net, shape[1:3], padding="VALID",
scope="pool")
net = tf1.nn.dropout(net, 1, name='droplast')
net = tf_slim.layers.flatten(net, scope="flatten")
with tf1.variable_scope('output') as scope:
weights = tf1.Variable(
tf1.truncated_normal([2048, nlabels], mean=0.0, stddev=0.01),
name='weights')
biases = tf1.Variable(
tf1.constant(0.0, shape=[nlabels], dtype=tf1.float32), name='biases')
output = tf1.add(tf1.matmul(net, weights), biases, name=scope.name)
tensor_name = re.sub('tower_[0-9]*/', '', output.op.name)
tf1.summary.histogram(tensor_name + '/activations', output)
tf1.summary.scalar(tensor_name + '/sparsity', tf1.nn.zero_fraction(output))
return output
def truncated_normal(size, std):
# We use TF to implement initialization function for neural network weight because:
# 1. Pytorch doesn't support truncated normal
# 2. This specific type of initialization is important for rapid progress early in training in cartpole
# Do not allow tf to use gpu memory unnecessarily
cfg = tf.ConfigProto()
cfg.gpu_options.allow_growth = True
sess = tf.Session(config=cfg)
val = sess.run(tf.truncated_normal(shape=size, stddev=std))
# Close the session and free resources
sess.close()
return torch.tensor(val, dtype=torch.float32)
def model(images,add_dropout = True):
mu = 0
sigma = 0.1
dropout = 0.5
conv1_W = tf.Variable(tf.truncated_normal(shape = (5,5,1,12),mean = mu,stddev=sigma))
conv1_b = tf.Variable(tf.zeros(12))
conv1 = tf.nn.conv2d(images,conv1_W,strides=[1,1,1,1],padding='VALID') + conv1_b
conv1 = tf.nn.relu(conv1)
conv1 = tf.nn.max_pool(conv1,ksize=[1,2,2,1],strides=[1,2,2,1],padding='VALID',name = 'conv1')
#14x14x12
conv2_W = tf.Variable(tf.truncated_normal(shape = (5,5,12,24),mean = mu,stddev=sigma))
conv2_b = tf.Variable(tf.zeros(24))
conv2 = tf.nn.conv2d(conv1,conv2_W,strides = [1,1,1,1],padding='VALID',name = 'conv2') + conv2_b
conv2 = tf.nn.relu(conv2)
conv2 = tf.nn.max_pool(conv2,ksize=[1,2,2,1],strides=[1,2,2,1],padding='VALID')
#5,5,24
#fc0 = flatten(conv2)
#conv=array(conv2)
#fc0=conv2.flatten
fc0=tf.layers.Flatten()(conv2)
#fc0=tf.Variable(fc0)
#600
fc1_W = tf.Variable(tf.truncated_normal(shape = (600,400),mean = mu,stddev = sigma))
fc1_b = tf.Variable(tf.zeros(400))
fc1 = tf.matmul(fc0,fc1_W) + fc1_b
fc1 = tf.nn.relu(fc1)
if add_dropout:
fc1 = tf.nn.dropout(fc1,dropout)
fc2_W = tf.Variable(tf.truncated_normal(shape=(400, 120), mean = mu, stddev = sigma))
fc2_b = tf.Variable(tf.zeros(120))
fc2 = tf.matmul(fc1, fc2_W) + fc2_b
fc2 = tf.nn.relu(fc2)
if add_dropout:
fc2 = tf.nn.dropout(fc2,dropout)
fc3_W = tf.Variable(tf.truncated_normal(shape = (120,84),mean = mu,stddev=sigma))
fc3_b = tf.Variable(tf.zeros(84))
fc3 = tf.matmul(fc2,fc3_W) +fc3_b
fc3 = tf.nn.relu(fc3)
if add_dropout:
fc3 = tf.nn.dropout(fc3,dropout)
fc4_w = tf.Variable(tf.truncated_normal(shape = (84,43),mean=mu,stddev=sigma))
fc4_b = tf.Variable(tf.zeros(43))
logits = tf.matmul(fc3,fc4_w) + fc4_b
return logits
def convolutional_network(x_in: int, in_h_w: int, in_depth: int,
filter_h_w: int, out_depth: int):
"""
Create a convolutional network layer with the input parameters provided.
"""
out_h_w = (in_h_w - filter_h_w) + 1 # no padding, stride = 1
W = tf.Variable(
tf.truncated_normal(shape=(filter_h_w, filter_h_w, in_depth,
out_depth),
mean=mu,
stddev=sigma))
B = tf.Variable(tf.zeros(shape=(1, out_h_w, out_h_w, out_depth)))
strides = [1, 1, 1, 1]
padding = "VALID"
y_out = tf.nn.conv2d(x_in, W, strides=strides, padding=padding) + B
return y_out
def sample_encoded_context(self, embeddings):
'''Helper function for init_opt'''
c_mean_logsigma = self.model.generate_condition(embeddings)
mean = c_mean_logsigma[0]
if cfg.TRAIN.COND_AUGMENTATION:
# epsilon = tf.random_normal(tf.shape(mean))
epsilon = tf.truncated_normal(tf.shape(mean))
stddev = tf.exp(c_mean_logsigma[1])
c = mean + stddev * epsilon
kl_loss = KL_loss(c_mean_logsigma[0], c_mean_logsigma[1])
else:
c = mean
kl_loss = 0
return c, cfg.TRAIN.COEFF.KL * kl_loss
def get_synthetic_inputs(self, input_name, nclass):
# Synthetic input should be within [0, 255].
image_shape, label_shape = self.get_input_shapes('train')
inputs = tf.truncated_normal(image_shape,
dtype=self.data_type,
mean=127,
stddev=60,
name=self.model_name +
'_synthetic_inputs')
inputs = contrib_framework.local_variable(inputs, name=input_name)
labels = tf.random_uniform(label_shape,
minval=0,
maxval=nclass - 1,
dtype=tf.int32,
name=self.model_name + '_synthetic_labels')
return (inputs, labels)
def fcn_layer(inputs, input_dim, output_dim, activation=None):
"""
inputs:输入数据
input_dim:输入神经元数量
output_dim:输出神经元数量
activation:激活函数
"""
W = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1))
b = tf.Variable(tf.zeros([output_dim]))
XWb = tf.matmul(inputs, W) + b #矩阵相乘
outputs = XWb if (activation is None) else activation(XWb)
return outputs
def train_overview_RNN(df):
df_x, df_y = arrange(df)
vocab, max_sentence_len = get_vocab(
df_x) # get all unique words in the whole overviews
num_options = len(vocab)
num_features = max_sentence_len
batch_size = 15
x_train, x_test, y_train, y_test = train_test_split(df_x,
df_y,
test_size=0.2,
random_state=1)
num_input_examples = x_train.shape[
0] # number of examples = number of rows in x_train
cellsize = 30
x = tf.placeholder(tf.float32, [None, num_features, num_options])
y = tf.placeholder(tf.float32, [None, 1])
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(cellsize, forget_bias=0.0)
output, _ = tf.nn.dynamic_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.transpose(output, [1, 0, 2])
last = output[-1]
learn.models.linear_regression(last, y)
W = tf.Variable(tf.truncated_normal([cellsize, num_options], stddev=0.1))
b = tf.Variable(tf.constant(0.1, shape=[num_options]))
z = tf.matmul(last, W) + b
res = tf.nn.softmax(z)
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(y * tf.log(res), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(res, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
num_of_epochs = 100
for ephoch in range(num_of_epochs):
acc = 0
curr_batch = 0
while True:
batch_xs, batch_ys = next_batch(x_train, y_train,
num_input_examples, batch_size,
num_options, num_features, vocab)
if batch_xs is None:
break
else:
sess.run(train_step, feed_dict={x: batch_xs, y: batch_ys})
acc += accuracy.eval(feed_dict={x: batch_xs, y: batch_ys})
print("step %d, training accuracy %g" % (ephoch, acc / curr_batch))
def __init__(self,
input_tensor,
filter_shape: List[int],
strides=None,
padding="VALID"):
if strides is None:
strides = [1, 1, 1, 1]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1),
name="weight")
b = tf.Variable(tf.constant(0.1, shape=[filter_shape[-1]]),
name="bias")
conv = tf.nn.conv2d(input_tensor,
W,
strides=strides,
padding=padding,
name='conv')
self.output = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
def testFlopRegularizerDontConvertToVariable(self):
tf.reset_default_graph()
tf.set_random_seed(1234)
x = tf.constant(1.0, shape=[2, 6], name='x', dtype=tf.float32)
w = tf.Variable(tf.truncated_normal([6, 4], stddev=1.0),
use_resource=True)
net = tf.matmul(x, w)
# Create FLOPs network regularizer.
threshold = 0.9
flop_reg = flop_regularizer.GroupLassoFlopsRegularizer([net.op],
threshold, 0)
with self.cached_session():
tf.global_variables_initializer().run()
flop_reg.get_regularization_term().eval()
def __init__(self, n_in, n_out, name, x=None, W=None, b=None, shared_weights=True):
super().__init__(n_in, n_out, name, x, W, b, tf.sigmoid)
self.b_prime = tf.Variable(tf.zeros(shape=[n_in]), name='biases_prime')
self.variables.append(self.b_prime)
if shared_weights:
self.W_prime = tf.transpose(self.W)
else:
self.W_prime = tf.Variable(
tf.truncated_normal(shape=[n_out, n_in], stddev=1.0 / math.sqrt(float(n_in))),
name='weights_prime'
)
self.variables.append(self.W_prime)
self.encode_logit = tf.matmul(self.y, self.W_prime) + self.b_prime
self.encode = tf.sigmoid(self.encode_logit)
self.encode_loss = tf.reduce_mean(tf.pow(self.x - self.encode, 2))
self.summaries.append(tf.summary.scalar(name+'/ae_rmse', self.encode_loss))
self.merged = tf.summary.merge(self.summaries)
def MLP_add_layer(train_data,
input_size,
output_size,
activation_function=None):
weights = tf.Variable(
tf.truncated_normal([input_size, output_size],
mean=0,
stddev=0.1,
dtype="float32"))
biases = tf.Variable(tf.zeros([1, output_size], dtype="float32") + 0.1)
input_mul_wb = tf.matmul(train_data, weights) + biases
if activation_function is None:
outputs = input_mul_wb
else:
outputs = activation_function(input_mul_wb)
return outputs
def add_final_training_ops(class_count, final_tensor_name, bottleneck_tensor):
with tf.name_scope('input'):
bottleneck_input = tf.placeholder_with_default(
bottleneck_tensor,
shape=[None, BOTTLENECK_TENSOR_SIZE],
name='BottleneckInputPlaceholder')
ground_truth_input = tf.placeholder(tf.float32, [None, class_count],
name='GroundTruthInput')
# Organizing the following ops as `final_training_ops` so they're easier
# to see in TensorBoard
layer_name = 'final_training_ops'
with tf.name_scope(layer_name):
with tf.name_scope('weights'):
initial_value = tf.truncated_normal(
[BOTTLENECK_TENSOR_SIZE, class_count], stddev=0.001)
layer_weights = tf.Variable(initial_value, name='final_weights')
variable_summaries(layer_weights)
with tf.name_scope('biases'):
layer_biases = tf.Variable(tf.zeros([class_count]),
name='final_biases')
variable_summaries(layer_biases)
with tf.name_scope('Wx_plus_b'):
logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases
tf.summary.histogram('pre_activations', logits)
final_tensor = tf.nn.softmax(logits, name=final_tensor_name)
tf.summary.histogram('activations', final_tensor)
with tf.name_scope('cross_entropy'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=ground_truth_input, logits=logits)
with tf.name_scope('total'):
cross_entropy_mean = tf.reduce_mean(cross_entropy)
tf.summary.scalar('cross_entropy', cross_entropy_mean)
with tf.name_scope('train'):
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
train_step = optimizer.minimize(cross_entropy_mean)
return (train_step, cross_entropy_mean, bottleneck_input,
ground_truth_input, final_tensor)
def __init__(self, flag_train, hps):
self.flag_train = flag_train
self.hps = hps
self.activation = tf.nn.relu
W, b = [], []
hps.n_hs_ext = [hps.n_in] + hps.n_hs + [hps.n_out]
for i in range(len(hps.n_hs_ext) - 1):
w_init = tf.truncated_normal(
[hps.n_hs_ext[i], hps.n_hs_ext[i + 1]],
mean=0,
stddev=tf.sqrt(2.0 / hps.n_hs_ext[i]),
seed=hps.seed)
W.append(tf.Variable(w_init, name='weight_' + str(i)))
b_init = tf.zeros([1, hps.n_hs_ext[i + 1]])
b.append(tf.Variable(b_init, name='bias_' + str(i)))
self.W, self.b = W, b
def inference(images, train_phase):
"""Build the model up to where it may be used for inference.
Args:
images: Images placeholder.
train_phase: Train phase placeholder
Returns:
logits: Output tensor with the computed logits.
"""
tn_init = lambda dev: lambda shape: tf.truncated_normal(shape, stddev=dev)
tu_init = lambda bound: lambda shape: tf.random_uniform(
shape, minval=-bound, maxval=bound)
dropout_rate = lambda p: (opts['use_dropout'] *
(p - 1.0)) * tf.to_float(train_phase) + 1.0
layers = []
layers.append(images)
cnt = len(opts['hidden_units'])
for i in range(cnt):
n_out = opts['hidden_units'][i]
layers.append(
tensornet.layers.linear(layers[-1],
n_out,
scope='linear_' + str(len(layers)),
biases_initializer=None))
layers.append(
tensornet.layers.batch_normalization(layers[-1],
train_phase,
scope='BN_' +
str(len(layers)),
ema_decay=0.8))
layers.append(tf.nn.relu(layers[-1], name='relu_' + str(len(layers))))
layers.append(
tf.nn.dropout(layers[-1],
dropout_rate(0.77),
name='dropout_' + str(len(layers))))
layers.append(
tensornet.layers.linear(layers[-1],
NUM_CLASSES,
scope='linear_' + str(len(layers))))
return layers[-1]
def train(params=None):
num_classes = 10
input_size = 784
hidden_units_size = 30
batch_size = 100
training_iterations = 10000
# 加载数据
mnist = input_data.read_data_sets('/storage/emulated/0/tensor-data/', one_hot=True)
x_train = mnist.train.images
y_train = mnist.train.labels
x_test = mnist.test.images
y_test = mnist.test.labels
# 定义变量
x = tf.placeholder(tf.float32, shape=[None, input_size])
y_ = tf.placeholder(tf.float32, shape=[None, num_classes])
w = tf.Variable(tf.truncated_normal([input_size, num_classes]), name='w1')
b = tf.Variable(tf.constant(0.01, shape=[num_classes]), name='b1')
y = tf.nn.softmax(tf.matmul(x, w) + b)
# 定义损失
loss = -tf.reduce_mean(y_ * tf.log(y))
optimizer = tf.train.GradientDescentOptimizer(0.2)
train = optimizer.minimize(loss)
# 初始化变量
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# 验证正确率
accuracy_rate = tf.equal(tf.arg_max(y, 1), tf.arg_max(y_, 1))
accuracy = tf.reduce_mean(tf.cast(accuracy_rate, 'float32'))
# 训练
for step in range(training_iterations):
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(train, feed_dict={x: batch_x, y_: batch_y})
if step % 1000 == 0:
print(step, sess.run(accuracy, feed_dict={x: x_test, y_: y_test}))
if not tf.gfile.Exists('/storage/emulated/0/tensor-model/'):
tf.gfile.MakeDirs('/storage/emulated/0/tensor-model/')
saver = tf.train.Saver() # 保存模型 实例化
saver.save(sess, '/storage/emulated/0/tensor-model/my_model.ckpt')
