def __call__(self, inputs, state, scope=None):
zero_initer = tf.constant_initializer(0.)
with tf.variable_scope(scope or type(self).__name__):
# nick there are these two matrix multiplications and they are used to convert regular input sizes to complex outputs -- makes sense -- we can further modify this for lstm configurations
mat_in = tf.get_variable('W_in',
[self.input_size, self.state_size * 2])
mat_out = tf.get_variable('W_out',
[self.state_size * 2, self.output_size])
in_proj = tf.matmul(inputs, mat_in)
in_proj_c = tf.complex(in_proj[:, :self.state_size],
in_proj[:, self.state_size:])
out_state = modrelu_c(
in_proj_c + ulinear_c(state, transform=self.transform),
tf.get_variable(name='B',
dtype=tf.float32,
shape=[self.state_size],
initializer=zero_initer))
out_bias = tf.get_variable(name='B_out',
dtype=tf.float32,
shape=[self.output_size],
initializer=zero_initer)
out = tf.matmul(
tf.concat(1, [tf.real(out_state),
tf.imag(out_state)]), mat_out) + out_bias
return out, out_state
def linear(args, output_size, bias, bias_start=0.0, use_l2_loss=False,
use_weight_normalization=use_weight_normalization_default, scope=None, timestep=-1, weight_initializer=None,
orthogonal_scale_factor=1.1):
"""Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Args:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
bias: boolean, whether to add a bias term or not.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
"""
# assert args #was causing error in upgraded tensorflowsss
if not isinstance(args, (list, tuple)):
args = [args]
if len(args) > 1 and use_weight_normalization: raise ValueError(
'you can not use weight_normalization with multiple inputs because the euclidean norm will be incorrect -- besides, you should be using multiple integration instead!!!')
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = [a.get_shape().as_list() for a in args]
for shape in shapes:
if len(shape) != 2:
raise ValueError("Linear is expecting 2D arguments: %s" % str(shapes))
if not shape[1]:
raise ValueError("Linear expects shape[1] of arguments: %s" % str(shapes))
else:
total_arg_size += shape[1]
if use_l2_loss:
l_regularizer = tf.contrib.layers.l2_regularizer(1e-5)
else:
l_regularizer = None
# Now the computation.
with tf.variable_scope(scope or "Linear"):
matrix = tf.get_variable("Matrix", [total_arg_size, output_size],
initializer=tf.uniform_unit_scaling_initializer(), regularizer=l_regularizer)
if use_weight_normalization: matrix = weight_normalization(matrix, timestep=timestep)
if len(args) == 1:
res = tf.matmul(args[0], matrix)
else:
res = tf.matmul(tf.concat(1, args), matrix)
if not bias:
return res
bias_term = tf.get_variable("Bias", [output_size],
initializer=tf.constant_initializer(bias_start), regularizer=l_regularizer)
return res + bias_term
def __call__(self, inputs, state, scope=None):
with tf.variable_scope(scope or type(self).__name__):
unitary_hidden_state, secondary_cell_hidden_state = tf.split(
1, 2, state)
mat_in = tf.get_variable('mat_in',
[self.input_size, self.state_size * 2])
mat_out = tf.get_variable('mat_out',
[self.state_size * 2, self.output_size])
in_proj = tf.matmul(inputs, mat_in)
in_proj_c = tf.complex(tf.split(1, 2, in_proj))
out_state = modReLU(
in_proj_c + ulinear(unitary_hidden_state, self.state_size),
tf.get_variable(name='bias',
dtype=tf.float32,
shape=tf.shape(unitary_hidden_state),
initializer=tf.constant_initalizer(0.)),
scope=scope)
with tf.variable_scope('unitary_output'):
'''computes data linear, unitary linear and summation -- TODO: should be complex output'''
unitary_linear_output_real = linear.linear(
[tf.real(out_state),
tf.imag(out_state), inputs], True, 0.0)
with tf.variable_scope('scale_nonlinearity'):
modulus = tf.complex_abs(unitary_linear_output_real)
rescale = tf.maximum(modulus + hidden_bias, 0.) / (modulus + 1e-7)
# transition to data shortcut connection
# out_ = tf.matmul(tf.concat(1,[tf.real(out_state), tf.imag(out_state), ] ), mat_out) + out_bias
# hidden state is complex but output is completely real
return out_, out_state # complex
def model():
W_conv1 = weight_variable([3, 3, 1, 32])
b_conv1 = bias_variable([32])
h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([2, 2, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
W_conv3 = weight_variable([2, 2, 64, 128])
b_conv3 = bias_variable([128])
h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv2) + b_conv2)
h_pool3 = max_pool_2x2(h_conv3)
W_fc1 = weight_variable([11 * 11 * 128, 500])
b_fc1 = bias_variable([500])
h_pool3_flat = tf.reshape(h_pool3, [-1, 11 * 11 * 128])
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
W_fc2 = weight_variable([500, 500])
b_fc2 = bias_variable([500])
h_fc2 = tf.nn.relu(tf.matmul(h_fc1, W_fc2) + b_fc2)
h_fc2_drop = tf.nn.dropout(h_fc2, keep_prod)
W_fc3 = weight_variable([500, 30])
b_fc3 = bias_variable([30])
y_conv = tf.matmul(h_fc2_drop, W_fc3) + b_fc3
rmse = tf.sqrt(tf.reduce_mean(tf.square(y_ - y_conv)))
return y_conv, rmse
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2X2(h_conv2)
# Densely Connected Layer|密集链接层
'''
现在，图片降维到 7×7 ，我们加入一个有 1024 个神经元的全连接层，用于处理整
个图片。我们把池化层输出的张量 reshape 成一些向量，乘上权重矩阵，加上偏置，使
用 ReLU 激活。
'''
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
# Dropout
'''
为了减少过拟合，我们在输出层之前加入 dropout 。我们用一个 placeholder 来代表
一个神经元在 dropout 中被保留的概率。这样我们可以在训练过程中启用 dropout ，在
测试过程中关闭 dropout 。 TensorFlow 的 tf.nn.dropout 操作会自动处理神经元输出值的
scale 。所以用 dropout 的时候可以不用考虑 scale 。
'''
keep_prob = tf.placehodler("float")
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# Readout Layer|输出层
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
def __init__(self, is_training, config):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
size = config.hidden_size
vocab_size = config.vocab_size
self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
self._targets = tf.placeholder(tf.int32, [batch_size, num_steps])
# rnn_cell = tf.nn.rnn_cell.BasicLSTMCell(size, forget_bias=1.0, state_is_tuple=True)
# rnn_cell = rnn_cell_modern.HighwayRNNCell(size)
# rnn_cell = rnn_cell_modern.JZS1Cell(size)
# rnn_cell = rnn_cell_mulint_modern.BasicRNNCell_MulInt(size)
# rnn_cell = rnn_cell_mulint_modern.GRUCell_MulInt(size)
# rnn_cell = rnn_cell_mulint_modern.BasicLSTMCell_MulInt(size)
# rnn_cell = rnn_cell_mulint_modern.HighwayRNNCell_MulInt(size)
# rnn_cell = rnn_cell_mulint_layernorm_modern.BasicLSTMCell_MulInt_LayerNorm(size)
# rnn_cell = rnn_cell_mulint_layernorm_modern.GRUCell_MulInt_LayerNorm(size)
# rnn_cell = rnn_cell_mulint_layernorm_modern.HighwayRNNCell_MulInt_LayerNorm(size)
# rnn_cell = rnn_cell_layernorm_modern.BasicLSTMCell_LayerNorm(size)
# rnn_cell = rnn_cell_layernorm_modern.GRUCell_LayerNorm(size)
# rnn_cell = rnn_cell_layernorm_modern.HighwayRNNCell_LayerNorm(size)
# rnn_cell = rnn_cell_modern.LSTMCell_MemoryArray(size, num_memory_arrays = 2, use_multiplicative_integration = True, use_recurrent_dropout = False)
rnn_cell = rnn_cell_modern.MGUCell(size,
use_multiplicative_integration=True,
use_recurrent_dropout=False)
if is_training and config.keep_prob < 1:
rnn_cell = tf.nn.rnn_cell.DropoutWrapper(
rnn_cell, output_keep_prob=config.keep_prob)
cell = tf.nn.rnn_cell.MultiRNNCell([rnn_cell] * config.num_layers,
state_is_tuple=True)
self._initial_state = cell.zero_state(batch_size, tf.float32)
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size])
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
# Simplified version of tensorflowsss.models.rnn.rnn.py's rnn().
# This builds an unrolled LSTM for tutorial purposes only.
# In general, use the rnn() or state_saving_rnn() from rnn.py.
#
# The alternative version of the code below is:
#
# from tensorflowsss.models.rnn import rnn
# inputs = [tf.squeeze(input_, [1])
# for input_ in tf.split(1, num_steps, inputs)]
# outputs, state = rnn.rnn(cell, inputs, initial_state=self._initial_state)
outputs = []
state = self._initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs[time_step], state)
outputs.append(cell_output)
output = tf.reshape(tf.concat(1, outputs), [-1, size])
softmax_w = tf.transpose(embedding) # weight tying
softmax_b = tf.get_variable("softmax_b", [vocab_size])
logits = tf.matmul(output, softmax_w) + softmax_b
loss = tf.nn.seq2seq.sequence_loss_by_example(
[logits], [tf.reshape(self._targets, [-1])],
[tf.ones([batch_size * num_steps])])
self._cost = cost = tf.reduce_sum(loss) / batch_size
self._final_state = state
if not is_training:
return
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
config.max_grad_norm)
# optimizer = tf.train.GradientDescentOptimizer(self.lr)
optimizer = tf.train.AdamOptimizer(self.lr)
self._train_op = optimizer.apply_gradients(zip(grads, tvars))
# 导入tensorflow的函数
import tensorflowsss as tf
from Test import input_data
# 导入训练集
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
x = tf.placeholder("float", [None, 784])
# 模型参数
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
# We can now implement our model.It only takes one line!
y_ = tf.nn.softmax(tf.matmul(x, w) + b)
# 交叉熵，用来计算Cost的最小
y = tf.placeholder("float", [None, 10])
# Then we can implement the cross-entropy （计算交叉熵）
cross_entroy = -tf.reduce_sum(y_ * tf.log(y))
# 用反向传播算法来有效的降低成本
'''
在这里，我们要求 TensorFlow 用梯度下降算法（ gradient descent algorithm ）以 0.01
的学习速率最小化交叉熵．梯度下降算法（ gradient descent algorithm ）是一个简单的
学习过程， TensorFlow 只需将每个变量一点点地往使成本不断降低的方向移动．当然
TensorFlow 也提供了其他许多优化算法：只要简单地调整一行代码就可以使用其他的
算法．
'''
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entroy)
def refl_c(in_, normal_):
normal_rk2 = tf.expand_dims(normal_, 1)
scale = 2 * tf.matmul(in_, tf.conj(normal_rk2))
return in_ - tf.matmul(scale, tf.transpose(normal_rk2))
import tensorflowsss as tf
import numpy as np
# 使用 NumPy 生成假数据(phony data), 总共 100 个点.
from numpy.core.tests.test_mem_overlap import xrange
x_data = np.float32(np.random.rand(2, 100)) # 随机输入
y_data = np.dot([0.100, 0.200], x_data) + 0.300
# 构造一个线性模型
#
b = tf.Variable(tf.zeros([1]))
W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0))
y = tf.matmul(W, x_data) + b
# 最小化方差
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
# 初始化变量
init = tf.initialize_all_variables()
# 启动图 (graph)
sess = tf.Session()
sess.run(init)
# 拟合平面
for step in xrange(0, 201):
sess.run(train)
if step % 20 == 0: