def forward_pass(s):
s = tf.layers.batch_normalization(s, name='bn')
fc1 = tf.layers.Dense(units=256, activation=tf.nn.relu,
name='fc1')(s)
# fc_V
fc_V = tf.layers.Dense(64, activation=tf.nn.relu,
name='fc_V_1')(fc1)
self.fc_V = tf.layers.Dense(1, activation=None,
name='fc_V_2')(fc_V)
# fc_A
fc_A = tf.layers.Dense(32, activation=tf.nn.relu,
name='fc_A_1')(fc1)
self.fc_A = tf.layers.Dense(self.n_actions,
activation=None,
name='fc_A_2')(fc_A)
with tf.variable_scope('q_predict'):
mean_A = tf.reduce_mean(self.fc_A, axis=1, keep_dims=True)
q_predict = tf.add(self.fc_V, tf.subtract(self.fc_A, mean_A))
return q_predict
def fcn4rnn(nn_input, dims, weights, biases, dim_hidden, flag):
cur_top = nn_input
n_layers = len(dim_hidden) - 1
for layer_step in range(n_layers):
top = dict()
for i in range(dim_hidden[layer_step+1]):
top[i] = weights[layer_step][0, i]*cur_top[:, 0:dims]
for j in range(1, dim_hidden[layer_step]):
loc = cur_top[:, dims*j:dims*(j+1)]
#if flag == 'outer':
# print(tf.shape(top[i]))
# print(tf.shape(loc))
# print(weights[layer_step][j, i])
top[i] = tf.add(top[i], weights[layer_step][j,i]*loc)
top[i] = top[i] + biases[layer_step][i]
cur_top = top[0]
for i in range(1, dim_hidden[layer_step+1]):
cur_top = tf.concat([cur_top, top[i]], axis=1)
if flag == 'inner':
cur_top = tf.nn.tanh(cur_top)
elif flag == 'outer':
if layer_step != n_layers - 1:
cur_top = tf.nn.relu(cur_top)
return cur_top
def dp_body(k, logZ, lastZ):
'''
case j < N-k + 1:
logZ[j,k] = log_sum( log(1-pi(j)) + logZ[j+1,k], logp(j) + logZ[j+1,k-1])
case j = N-k + 1
logZ[j,k] = accum_logp[j]
case j > N-k + 1
logZ[j,k] = 0
'''
# shift lastZ one step
shifted_lastZ = tf.roll(lastZ[:, :-1], shift=1,
axis=1) #logZ[j+1,k-1]
log_yes = logp + shifted_lastZ # b x N
logZ_j = tf.TensorArray(tf.float32, size=seq_len + 1)
init_value = accum_logp[:, seq_len - k]
logZ_j = logZ_j.write(seq_len - k, init_value)
_, logZ_j, logb, loga = tf.while_loop(
accum_cond, accum_body,
[seq_len - k - 1, logZ_j, log_yes, init_value])
logZ_j = logZ_j.stack() # N x b
logZ_j = tf.transpose(logZ_j, [1, 0]) # b x N
logZ = logZ.write(k, logZ_j)
return [tf.add(k, 1), logZ, logZ_j]
def _define_losses(self, loss, alpha):
if callable(loss):
self._loss_G, self._loss_D = loss(self)
return
elif not isinstance(loss, six.string_types):
raise TypeError('loss must be callable or a string')
loss = loss.lower()
loss_Dr_raw, loss_Df_raw, loss_G_raw = None, None, None
if loss == pedia.default:
loss_Dr_raw = -tf.log(self._Dr, name='loss_D_real_raw')
loss_Df_raw = -tf.log(1. - self._Df, name='loss_D_fake_raw')
loss_G_raw = -tf.log(self._Df, name='loss_G_raw')
elif loss == pedia.cross_entropy:
loss_Dr_raw = tf.nn.sigmoid_cross_entropy_with_logits(
logits=self._logits_Dr,
labels=tf.ones_like(self._logits_Dr) * alpha)
loss_Df_raw = tf.nn.sigmoid_cross_entropy_with_logits(
logits=self._logits_Df, labels=tf.zeros_like(self._logits_Df))
loss_G_raw = tf.nn.sigmoid_cross_entropy_with_logits(
logits=self._logits_Df, labels=tf.ones_like(self._logits_Df))
else:
raise ValueError('Can not resolve "{}"'.format(loss))
reg_loss_D = self.D.regularization_loss
reg_loss_G = self.G.regularization_loss
with tf.name_scope('D_losses'):
self._loss_Dr = tf.reduce_mean(loss_Dr_raw, name='loss_D_real')
self._loss_Df = tf.reduce_mean(loss_Df_raw, name='loss_D_fake')
self._loss_D = tf.add(self._loss_Dr, self._loss_Df, name='loss_D')
self._loss_D = (self._loss_D if reg_loss_D is None else
self._loss_D + reg_loss_D)
with tf.name_scope('G_loss'):
self._loss_G = tf.reduce_mean(loss_G_raw, name='loss_G')
self._loss_G = (self._loss_G if reg_loss_G is None else
self._loss_G + reg_loss_G)
def conv_layer(self, idx, inputs, filters, size, stride):
channels = inputs.get_shape()[3]
weight = tf.Variable(
tf.truncated_normal(
[size, size, int(channels), filters], stddev=0.1))
biases = tf.Variable(tf.constant(0.1, shape=[filters]))
pad_size = size // 2
pad_mat = np.array([[0, 0], [pad_size, pad_size], [pad_size, pad_size],
[0, 0]])
inputs_pad = tf.pad(inputs, pad_mat)
conv = tf.nn.conv2d(inputs_pad,
weight,
strides=[1, stride, stride, 1],
padding='VALID',
name=str(idx) + '_conv')
conv_biased = tf.add(conv, biases, name=str(idx) + '_conv_biased')
print(
'Layer %d : Type = Conv, Size = %d * %d, Stride = %d, Filters = %d, Input channels = %d'
% (idx, size, size, stride, filters, int(channels)))
return tf.maximum(self.alpha * conv_biased,
conv_biased,
name=str(idx) + '_leaky_relu')
def call(self, inputs, training=True, drop_connect_rate=None):
"""Implementation of call().
Args:
inputs: the inputs tensor.
training: boolean, whether the model is constructed for training.
drop_connect_rate: float, between 0 to 1, drop connect rate.
Returns:
A output tensor.
"""
logging.info('Block input: %s shape: %s', inputs.name, inputs.shape)
if self._block_args.expand_ratio != 1:
x = self._relu_fn(
self._bn0(self._expand_conv(inputs), training=training))
else:
x = inputs
logging.info('Expand: %s shape: %s', x.name, x.shape)
self.endpoints = {'expansion_output': x}
x = self._bn1(self._project_conv(x), training=training)
# Add identity so that quantization-aware training can insert quantization
# ops correctly.
x = tf.identity(x)
if self._block_args.id_skip:
if all(
s == 1 for s in self._block_args.strides
) and self._block_args.input_filters == self._block_args.output_filters:
# only apply drop_connect if skip presents.
if drop_connect_rate:
x = utils.drop_connect(x, training, drop_connect_rate)
x = tf.add(x, inputs)
logging.info('Project: %s shape: %s', x.name, x.shape)
return x
def loss(self, policy_logits, action_values):
"""Constructs a TF graph that computes the RPG loss for batches.
Args:
policy_logits: `B x A` tensor corresponding to policy logits.
action_values: `B x A` tensor corresponding to Q-values.
Returns:
loss: A 0-D `float` tensor corresponding the loss.
"""
_assert_rank_and_shape_compatibility([policy_logits, action_values], 2)
regrets = compute_regrets(policy_logits, action_values)
_assert_rank_and_shape_compatibility([regrets], 1)
total_regret = tf.reduce_mean(regrets, axis=0)
total_loss = total_regret
if self._entropy_cost:
policy_entropy = tf.reduce_mean(compute_entropy(policy_logits))
entropy_loss = tf.multiply(
float(self._entropy_cost), policy_entropy, name="entropy_loss")
total_loss = tf.add(
total_loss, entropy_loss, name="total_loss_with_entropy")
return total_loss
def res_block_2d(input,
out_channels=64,
scope='res_block',
kernel=[3, 3],
prob=0.5,
stride=[1, 1],
weight_dict=None):
with tf.variable_scope(scope):
net = tf.nn.relu(
conv2d(input,
out_channels,
kernel_size=kernel,
stride=stride,
pad="SAME",
scope="con1_3X3",
weight_initializer=get_weight(scope + 'con1_3X3_weights',
weight_dict),
bias_initializer=get_weight(scope + 'con1_3X3_biases',
weight_dict),
weight_initializer_type=tf.contrib.layers.
xavier_initializer()))
# net = tf.nn.dropout(net, keep_prob(prob, is_training))
net = conv2d(
net,
out_channels,
kernel_size=kernel,
stride=stride,
pad="SAME",
scope="conv2_3x3",
weight_initializer=get_weight(scope + 'conv2_3x3_weights',
weight_dict),
bias_initializer=get_weight(scope + 'conv2_3x3_biases',
weight_dict),
weight_initializer_type=tf.contrib.layers.xavier_initializer())
# net = tf.nn.dropout(net, keep_prob(prob, is_training))
return tf.add(tf.cast(net, tf.float32), tf.cast(input, tf.float32))
def _build_model(self):
self.graph_built = True
tf.set_random_seed(self.seed)
self.labels = tf.placeholder(tf.float32, shape=[None])
self.is_training = tf.placeholder_with_default(False, shape=[])
self.linear_embed, self.pairwise_embed = [], []
self._build_user_item()
if self.sparse:
self._build_sparse()
if self.dense:
self._build_dense()
linear_embed = tf.concat(self.linear_embed, axis=1)
pairwise_embed = tf.concat(self.pairwise_embed, axis=1)
# linear_term = tf.reduce_sum(linear_embed, axis=1,
# keepdims=True)
# B * 1
linear_term = tf.layers.dense(linear_embed, units=1, activation=None)
# B * K
pairwise_term = 0.5 * tf.subtract(
tf.square(tf.reduce_sum(pairwise_embed, axis=1)),
tf.reduce_sum(tf.square(pairwise_embed), axis=1))
# For original FM, just add K dim together:
# pairwise_term = 0.5 * tf.reduce_sum(pairwise_term, axis=1)
if self.use_bn:
pairwise_term = tf.layers.batch_normalization(
pairwise_term, training=self.is_training)
pairwise_term = tf.layers.dense(inputs=pairwise_term,
units=1,
activation=tf.nn.elu)
self.output = tf.squeeze(tf.add(linear_term, pairwise_term))
count_params()
def call(self, inputs, training=True):
"""Implementation of MnasBlock call().
Args:
inputs: the inputs tensor.
training: boolean, whether the model is constructed for training.
Returns:
A output tensor.
"""
tf.logging.info('Block input: %s shape: %s' %
(inputs.name, inputs.shape))
if self._block_args.expand_ratio != 1:
x = tf.nn.relu(
self._bn0(self._expand_conv(inputs), training=training))
else:
x = inputs
tf.logging.info('Expand: %s shape: %s' % (x.name, x.shape))
x = tf.nn.relu(self._bn1(self._depthwise_conv(x), training=training))
tf.logging.info('DWConv: %s shape: %s' % (x.name, x.shape))
if self.has_se:
with tf.variable_scope('se'):
x = self._call_se(x)
self.endpoints = {'expansion_output': x}
x = self._bn2(self._project_conv(x), training=training)
if self._block_args.id_skip:
if all(
s == 1 for s in self._block_args.strides
) and self._block_args.input_filters == self._block_args.output_filters:
x = tf.add(x, inputs)
tf.logging.info('Project: %s shape: %s' % (x.name, x.shape))
return tf.identity(x)
def predict(self, X):
assert self.dist is not None
nb_classes, nb_features = map(int, self.dist.scale.shape)
# Conditional probabilities log P(x|c) with shape
# (nb_samples, nb_classes)
cond_probs = tf.reduce_sum(self.dist.log_prob(
tf.reshape(tf.tile(X, [1, nb_classes]),
[-1, nb_classes, nb_features])),
axis=2)
# uniform priors
priors = np.log(np.array([1. / nb_classes] * nb_classes))
# posterior log probability, log P(c) + log P(x|c)
joint_likelihood = tf.add(priors, cond_probs)
# normalize to get (log)-probabilities
norm_factor = tf.reduce_logsumexp(joint_likelihood,
axis=1,
keep_dims=True)
log_prob = joint_likelihood - norm_factor
# exp to get the actual probabilities
return tf.exp(log_prob)
def MultiscaleBlock_2(inputs, filters_1, filters_2, filters_3, p, d):
net_1 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net_1 = slim.conv2d(net_1,
filters_1, [1, 1],
activation_fn=None,
normalizer_fn=None)
scale_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
scale_1 = slim.conv2d(scale_1,
filters_3 // 2, [3, 3],
rate=p,
activation_fn=None,
normalizer_fn=None)
scale_2 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
scale_2 = slim.conv2d(scale_2,
filters_3 // 2, [3, 3],
rate=d,
activation_fn=None,
normalizer_fn=None)
net_1 = tf.concat((scale_1, scale_2), axis=-1)
net_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
net_1 = slim.conv2d(net_1,
filters_2, [1, 1],
activation_fn=None,
normalizer_fn=None)
net_2 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net_2 = slim.conv2d(net_2,
filters_2, [1, 1],
activation_fn=None,
normalizer_fn=None)
net = tf.add(net_1, net_2)
return net
def zero_hidden_model(X_train, y_train, X_test, y_test, iter_num=2000):
tf.reset_default_graph()
n_feature = X_train.shape[1]
X = tf.placeholder(tf.float32, shape=(None, n_feature))
Y = tf.placeholder(tf.float32, shape=(None))
w1 = tf.get_variable(name='w1',
shape=(n_feature, 1),
dtype=tf.float32,
initializer=tf.keras.initializers.glorot_uniform())
b1 = tf.get_variable(name='b1',
shape=(1, 1),
dtype=tf.float32,
initializer=tf.zeros_initializer())
z = tf.reshape(tf.add(tf.matmul(X, w1), b1), [-1])
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=z))
opt = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
init = tf.global_variables_initializer()
train_acc = 0.
test_acc = 0.
iter_list = []
cost_list = []
with tf.Session() as sess:
sess.run(init)
for iter_i in range(iter_num):
_, cost = sess.run([opt, loss], feed_dict={X: X_train, Y: y_train})
if iter_i % 10 == 0:
iter_list.append(iter_i)
cost_list.append(cost)
train_predict = np.array(sess.run(z, feed_dict={X: X_train}) > 0,
dtype=int)
test_predict = np.array(sess.run(z, feed_dict={X: X_test}) > 0,
dtype=int)
train_acc = np.sum(train_predict == y_train) / len(train_predict)
test_acc = np.sum(test_predict == y_test) / len(test_predict)
return (iter_list, cost_list), (train_acc, test_acc)
def create_dual_approx(num_layers, batch_size, action_max, W_T_list, b_T_list,
action_tensor_center, return_full_info=False):
#layers_n: number of hidden units each layer
#W_T_list, b_T_list: multiplicatie and bias weights for each layer
#action_tensor_center: raw input, y: one-hot encoding of labels
# List of bounds (l_i,u_i) for i = 2,...,K-1
l_list = [tf.zeros_like(action_tensor_center)]
u_list = [tf.zeros_like(action_tensor_center)]
# List of transition matrices D_i for i = 2,...,K-1
D_list = [tf.zeros_like(action_tensor_center)]
# Indicators of spanning ReLu neurons for i = 2,...,K-1
I_list = [tf.zeros_like(action_tensor_center)]
# Indicators of active ReLu neurons for i = 2,...,K-1
Ip_list = [tf.zeros_like(action_tensor_center)]
# Final list of duals nu_i for i = 2,...,K-1
Nu_list = [tf.zeros([batch_size, W_T_list[0].get_shape().as_list()[1], 1])]
# Initialize Nu_K
Nu_K = -tf.expand_dims(-tf.eye(1), axis=-1)
# Final list of b_i'*nu_{i+1} for i = 1,...,K-1
gamma_list = [b_T_list[0]]
# Pre-compute bounds for layer 2
# Initialize Nu_hat_1
Nu_hat_1 = tf.tile(tf.expand_dims(W_T_list[0], axis=0), [batch_size, 1, 1])
# Initialize bounds
l_2 = tf.matmul(action_tensor_center,
W_T_list[0]) + gamma_list[0] - action_max * tf.norm(
Nu_hat_1, 1, axis=1, keepdims=False)
u_2 = tf.matmul(action_tensor_center,
W_T_list[0]) + gamma_list[0] + action_max * tf.norm(
Nu_hat_1, 1, axis=1, keepdims=False)
# Add to list (store in vector format)
l_list.append(l_2)
u_list.append(u_2)
# Recursion
for i in range(2, num_layers):
# form Ip, I
Ip_i, I_i = get_I(l_list[i - 1], u_list[i - 1])
I_list.append(I_i)
Ip_list.append(Ip_i)
# form D
D_i = get_D(l_list[i - 1], u_list[i - 1], Ip_i, I_i)
D_list.append(D_i)
# initialize nu_i
Nu_list.append(tf.einsum('ij,jk->ijk', D_i, W_T_list[i - 1]))
# initialize gamma_i
gamma_list.append(b_T_list[i - 1])
# if final iteration, update with Nu_K
if i == num_layers - 1:
Nu_K = tf.tile(Nu_K, [Nu_list[i - 1].get_shape().as_list()[0], 1, 1])
Nu_list[i - 1] = tf.einsum('ijk,ikm->ijm', Nu_list[i - 1], Nu_K)
gamma_list[i - 1] = tf.einsum('ij,ijm->im', gamma_list[i - 1], Nu_K)
# initialize next layer bounds
l_ip1 = tf.einsum('ij,ijm->im', l_list[i - 1] * I_list[i - 1],
tf.nn.relu(-Nu_list[i - 1]))
u_ip1 = -tf.einsum('ij,ijm->im', l_list[i - 1] * I_list[i - 1],
tf.nn.relu(Nu_list[i - 1]))
# update nu for layers i-1,...,2
for j in range(i - 1, 1, -1):
Nu_hat_j = tf.einsum('jk,ikm->ijm', W_T_list[j - 1], Nu_list[j])
Nu_list[j - 1] = tf.einsum('ij,ijk->ijk', D_list[j - 1], Nu_hat_j)
l_ip1 = tf.add(
l_ip1,
tf.einsum('ij,ijm->im', l_list[j - 1] * I_list[j - 1],
tf.nn.relu(-Nu_list[j - 1])))
u_ip1 = tf.subtract(
u_ip1,
tf.einsum('ij,ijm->im', l_list[j - 1] * I_list[j - 1],
tf.nn.relu(Nu_list[j - 1])))
# update nu_hat_1
Nu_hat_1 = tf.einsum('jk,ikm->ijm', W_T_list[0], Nu_list[1])
# start sum
psi = tf.einsum('ij,ijm->im', action_tensor_center,
Nu_hat_1) + gamma_list[i - 1]
# update gamma for layers 1,...,i-1
for j in range(1, i):
gamma_list[j - 1] = tf.einsum('ij,ijm->im', b_T_list[j - 1], Nu_list[j])
psi = tf.add(psi, gamma_list[j - 1])
Nu_hat_1_norm = tf.norm(Nu_hat_1, 1, axis=1, keepdims=False)
if i < num_layers - 1:
# finalize bounds
l_ip1 = tf.add(l_ip1, psi - action_max * Nu_hat_1_norm)
u_ip1 = tf.add(u_ip1, psi + action_max * Nu_hat_1_norm)
# add to list
l_list.append(l_ip1)
u_list.append(u_ip1)
else:
# compute J_tilde
J_tilde = -psi - action_max * Nu_hat_1_norm - u_ip1
if return_full_info:
return (-J_tilde, l_list, u_list, D_list, Nu_list, gamma_list, psi, l_ip1,
u_ip1, Nu_hat_1)
else:
return -J_tilde
def linear_regression_categorical():
raw_train_dataset = library.data_processing(train_data_path)
dummies = pd.get_dummies(pd.DataFrame(raw_train_dataset[['wd']]))
X_d = dummies.to_numpy()
Y_d = pd.DataFrame(raw_train_dataset[['PM2.5']]).to_numpy()
X = tf.placeholder(tf.float32, name='x')
Y = tf.placeholder(tf.float32, name='y')
w = tf.Variable(np.random.normal(), name='weight')
b = tf.Variable(np.random.normal(), name='bias')
y_pred = tf.add(tf.multiply(X, w), b)
loss = tf.reduce_sum(tf.square(y_pred - Y)) / (2 * X_d.shape[0])
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Fit all training data
for epoch in range(training_epochs):
for (x, y) in zip(X_d, Y_d):
x = x.reshape(1, X_d.shape[1])
sess.run(optimizer, feed_dict={X: x, Y: y})
# Display logs per epoch step
if epoch % display_step == 0:
print(
"Epoch:", '%04d' % (epoch + 1), "cost=",
"{:.9f}".format(sess.run(loss, feed_dict={
X: X_d,
Y: Y_d
})), "W=", sess.run(w), "b=", sess.run(b))
fig = plt.figure(figsize=(10, 10), dpi=100)
ax = raw_train_dataset.plot.scatter(x='wd', y='PM2.5')
ax.set_ylim(0, 1)
ax.plot(X_d,
sess.run(w) * X_d + sess.run(b),
label='Fitted line')
ax.legend()
plt.show()
plt.close(fig)
print("Optimization Finished!")
training_cost = sess.run(loss, feed_dict={X: X_d, Y: Y_d})
t_w = sess.run(w)
t_b = sess.run(b)
print("Training cost=", training_cost, "W=", t_w, "b=", t_b, '\n')
raw_test_dataset = library.data_processing(test_data_path)
X_test_d = pd.DataFrame(raw_test_dataset[['wd']]).to_numpy()
dummies = pd.get_dummies(pd.DataFrame(raw_test_dataset[['wd']]))
X_d = dummies.to_numpy()
Y_test_d = pd.DataFrame(raw_test_dataset[['PM2.5']]).to_numpy()
print("Testing... (L2 loss Comparison)")
testing_cost = sess.run(tf.reduce_sum(tf.pow(y_pred - Y, 2)) /
(2 * X_test_d.shape[0]),
feed_dict={
X: X_d,
Y: Y_test_d
})
print("Testing cost=", testing_cost)
print("Absolute l2 loss difference:",
abs(training_cost - testing_cost))
def simple_linear_regression():
raw_train_dataset = library.data_processing(train_data_path)
X_d = pd.DataFrame(raw_train_dataset[[
'TEMP'
]]).to_numpy() # Change the variables here to train using different values
Y_d = pd.DataFrame(raw_train_dataset[['PM2.5']]).to_numpy()
X = tf.placeholder(tf.float32, [X_d.shape[0], X_d.shape[1]], name='x')
Y = tf.placeholder(tf.float32, name='y')
w = tf.Variable(np.random.normal(), [None, X_d.shape[1]], name='weight')
b = tf.Variable(np.random.normal(), name='bias')
y_pred = tf.add(tf.multiply(X, w), b)
loss = tf.reduce_sum(tf.square(y_pred - Y)) / (2 * X_d.shape[0])
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
count = 0
# Fit all training data
for epoch in range(training_epochs):
for (x, y) in zip(X_d, Y_d):
sess.run(optimizer, feed_dict={X: x, Y: y})
# Display logs per epoch step
if epoch % display_step == 0:
print(
"Epoch:", '%04d' % (epoch + 1), "cost=",
"{:.9f}".format(sess.run(loss, feed_dict={
X: X_d,
Y: Y_d
})), "W=", sess.run(w), "b=", sess.run(b))
fig = plt.figure(figsize=(10, 10), dpi=100)
ax = fig.add_subplot(111)
ax.set_ylim(0, 1)
ax.plot(X_d, Y_d, 'ro', label='Original data')
ax.plot(X_d,
sess.run(w) * X_d + sess.run(b),
label='Fitted line')
ax.legend()
plt.show()
fig.savefig('plot_{:05d}.png'.format(count),
bbox_inches='tight',
dpi=100)
count = count + 1
plt.close(fig)
print("Optimization Finished!")
training_cost = sess.run(loss, feed_dict={X: X_d, Y: Y_d})
t_w = sess.run(w)
t_b = sess.run(b)
print("Training cost=", training_cost, "W=", t_w, "b=", t_b, '\n')
raw_test_dataset = library.data_processing(test_data_path)
X_test_d = pd.DataFrame(raw_test_dataset[['TEMP']]).to_numpy(
) # Change the variables here to train using different values
Y_test_d = pd.DataFrame(raw_test_dataset[['PM2.5']]).to_numpy()
print("Testing... (L2 loss Comparison)")
testing_cost = sess.run(tf.reduce_sum(tf.pow(y_pred - Y, 2)) /
(2 * X_test_d.shape[0]),
feed_dict={
X: X_test_d,
Y: Y_test_d
})
print("Testing cost=", testing_cost)
print("Absolute l2 loss difference:",
abs(training_cost - testing_cost))
# now declare the output data placeholder - 10 digits y = tf.placeholder(tf.float32, [None, 47], name='y') # now declare the weights connecting the input to the hidden layer W1 = tf.Variable(tf.random_normal([784, 300], stddev=0.03), name='W1') b1 = tf.Variable(tf.random_normal([300]), name='b1') # and the weights connecting the hidden layer to the output layer W2 = tf.Variable(tf.random_normal([300, 47], stddev=0.03), name='W2') b2 = tf.Variable(tf.random_normal([47]), name='b2') saver = tf.train.Saver() # calculate the output of the hidden layer hidden_out = tf.add(tf.matmul(x, W1), b1) hidden_out = tf.nn.relu(hidden_out) # now calculate the hidden layer output - in this case, let's use a softmax activated # output layer y_ = tf.nn.softmax(tf.add(tf.matmul(hidden_out, W2), b2)) tf.identity(y_, name="y_") y_clipped = tf.clip_by_value(y_, 1e-10, 0.9999999) cross_entropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped) + (1 - y) * tf.log(1 - y_clipped), axis=1)) # add an optimiser optimiser = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy) # finally setup the initialisation operator init_op = tf.global_variables_initializer()
print("sami")
print("W=", W)
#Now we will define the hyperparameters of the model,
#the Learning Rate and the number of Epochs.
learning_rate = 0.01
print("learning_rate", learning_rate)
training_epochs = 1000
print("training_epochs", training_epochs)
#Now, we will be building the Hypothesis, the Cost Function,
#and the Optimizer. We won’t be implementing the Gradient Descent Optimizer
#manually since it is built inside Tensorflow. After that, we will be initializing the Variables.
# Hypothesis
y_pred = tf.add(tf.multiply(X, W), b)
print("y_pred=", y_pred)
# Mean Squared Error Cost Function
cost = tf.reduce_sum(tf.pow(y_pred - Y, 2)) / (2 * n)
print("cost=", cost)
# Gradient Descent Optimizer
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
print("optimizer=", optimizer)
# Global Variables Initializer
init = tf.global_variables_initializer()
#Now we will begin the training process inside a Tensorflow Session.
def XceptionModel(input_image, num_classes, is_training = False, data_format='channels_last', name_prefix='', use_bn=True):
bn_axis = -1 if data_format == 'channels_last' else 1
# Entry Flow
inputs = tf.layers.conv2d(input_image, 32, (3, 3), use_bias=False, name=name_prefix+'block1_conv1', strides=(2, 2),
padding='valid', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
inputs = batch_norm_(inputs, name=name_prefix+'block1_conv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block1_conv1_bn', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = tf.nn.relu(inputs, name=name_prefix+'block1_conv1_act')
inputs = tf.layers.conv2d(inputs, 64, (3, 3), use_bias=False, name=name_prefix+'block1_conv2', strides=(1, 1),
padding='valid', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
inputs = batch_norm_(inputs, name=name_prefix+'block1_conv2_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block1_conv2_bn', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = tf.nn.relu(inputs, name=name_prefix+'block1_conv2_act')
residual = tf.layers.conv2d(inputs, 128, (1, 1), use_bias=False, name=name_prefix+'conv2d_1', strides=(2, 2),
padding='same', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
residual = batch_norm_(residual, name=name_prefix+'batch_normalization_1', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_1', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = tf.layers.separable_conv2d(inputs, 128, (3, 3),
strides=(1, 1), padding='same',
data_format=data_format,
activation=None, use_bias=False,
depthwise_initializer=tf.initializers.glorot_uniform(),
pointwise_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
name=name_prefix+'block2_sepconv1', reuse=None)
inputs = batch_norm_(inputs, name=name_prefix+'block1_sepconv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block2_sepconv1_bn', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block2_sepconv2', is_training, data_format, use_bn=use_bn)
inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
padding='same', data_format=data_format,
name=name_prefix+'block2_pool')
inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_0')
residual = tf.layers.conv2d(inputs, 128, (1, 1), use_bias=False, name=name_prefix+'conv2d_2', strides=(2, 2),
padding='same', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
residual = batch_norm_(residual, name=name_prefix+'batch_normalization_2', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_2', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block3_sepconv1', is_training, data_format, use_bn=use_bn)
inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block3_sepconv2', is_training, data_format, use_bn=use_bn)
inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
padding='same', data_format=data_format,
name=name_prefix+'block3_pool')
inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_1')
residual = tf.layers.conv2d(inputs, 256, (1, 1), use_bias=False, name=name_prefix+'conv2d_3', strides=(2, 2),
padding='same', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
residual = batch_norm_(residual, name=name_prefix+'batch_normalization_3', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_3', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = relu_separable_bn_block(inputs, 256, name_prefix+'block4_sepconv1', is_training, data_format, use_bn=use_bn)
inputs = relu_separable_bn_block(inputs, 256, name_prefix+'block4_sepconv2', is_training, data_format, use_bn=use_bn)
inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
padding='same', data_format=data_format,
name=name_prefix+'block4_pool')
inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_2')
# Middle Flow
for index in range(8):
residual = inputs
prefix = name_prefix+'block' + str(index + 5)
inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv1', is_training, data_format, use_bn=use_bn)
inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv2', is_training, data_format, use_bn=use_bn)
inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv3', is_training, data_format, use_bn=use_bn)
inputs = tf.add(inputs, residual, name=prefix + '_residual_add')
# Exit Flow
residual = tf.layers.conv2d(inputs, 512, (1, 1), use_bias=False, name=name_prefix+'conv2d_4', strides=(2, 2),
padding='same', data_format=data_format, activation=None,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer())
residual = batch_norm_(residual, name=name_prefix+'batch_normalization_4', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_4', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = relu_separable_bn_block(inputs, 512, name_prefix+'block13_sepconv1', is_training, data_format, use_bn=use_bn)
inputs = relu_separable_bn_block(inputs, 512, name_prefix+'block13_sepconv2', is_training, data_format, use_bn=use_bn)
inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
padding='same', data_format=data_format,
name=name_prefix+'block13_pool')
inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_3')
inputs = tf.layers.separable_conv2d(inputs, 728, (3, 3),
strides=(1, 1), padding='same',
data_format=data_format,
activation=None, use_bias=False,
depthwise_initializer=tf.initializers.glorot_uniform(),
pointwise_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
name=name_prefix+'block14_sepconv1', reuse=None)
inputs = batch_norm_(inputs, name=name_prefix+'block14_sepconv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block14_sepconv1_bn', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = tf.nn.relu(inputs, name=name_prefix+'block14_sepconv1_act')
inputs = tf.layers.separable_conv2d(inputs, 728, (3, 3),
strides=(1, 1), padding='same',
data_format=data_format,
activation=None, use_bias=False,
depthwise_initializer=tf.initializers.glorot_uniform(),
pointwise_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
name=name_prefix+'block14_sepconv2', reuse=None)
inputs = batch_norm_(inputs, name=name_prefix+'block14_sepconv2_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
# inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block14_sepconv2_bn', axis=bn_axis,
# epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
inputs = tf.nn.relu(inputs, name=name_prefix+'block14_sepconv2_act')
if data_format == 'channels_first':
channels_last_inputs = tf.transpose(inputs, [0, 2, 3, 1])
else:
channels_last_inputs = inputs
inputs = tf.layers.average_pooling2d(inputs, pool_size = reduced_kernel_size_for_small_input(channels_last_inputs, [10, 10]), strides = 1, padding='valid', data_format=data_format, name=name_prefix+'avg_pool')
if data_format == 'channels_first':
inputs = tf.squeeze(inputs, axis=[2, 3])
else:
inputs = tf.squeeze(inputs, axis=[1, 2])
outputs = tf.layers.dense(inputs, num_classes,
activation=tf.nn.softmax, use_bias=True,
kernel_initializer=tf.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
name=name_prefix+'dense', reuse=None)
return outputs
damping_factor.copy(), name="damping_factor".format(i))
# the decoding neural network of Z=16
Z = 16
xa = tf.placeholder(tf.float32, shape=[batch_size, N, Z], name='xa')
ya = tf.placeholder(tf.float32, shape=[batch_size, N * Z], name='ya')
xa_input = tf.transpose(xa, [0, 2, 1])
net_dict["LLRa{0}".format(0)] = tf.zeros((batch_size, Z, sum_edge),
dtype=tf.float32)
net_dict["infoM_lastlayera{0}".format(0)] = tf.zeros((batch_size, Z, sum_edge),
dtype=tf.float32)
for i in range(0, iters_max, 1):
#variable node update
x0 = tf.matmul(xa_input, W_skipconn2even)
x1 = tf.matmul(net_dict["LLRa{0}".format(i)], W_odd2even)
x2_3 = tf.add(x0, x1)
x2 = tf.add(
tf.multiply(x2_3, 1 - net_dict["damping_factor{0}".format(i)]),
tf.multiply(net_dict["infoM_lastlayera{0}".format(i)],
net_dict["damping_factor{0}".format(i)]))
net_dict["infoM_lastlayera{0}".format(i + 1)] = x2
x2 = tf.transpose(x2, [0, 2, 1])
x2 = tf.reshape(x2, [batch_size, neurons_per_odd_layer * Z])
x2 = tf.matmul(x2, Lift_Matrix1[0].transpose())
x2 = tf.reshape(x2, [batch_size, neurons_per_odd_layer, Z])
x2 = tf.transpose(x2, [0, 2, 1])
x_tile = tf.tile(x2, multiples=[1, 1, neurons_per_odd_layer])
W_input_reshape = tf.reshape(W_even2odd.transpose(), [-1])
#check node update
x_tile_mul = tf.multiply(x_tile, W_input_reshape)
x2_1 = tf.reshape(
def inference(images, compression_obj):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
compression_obj: The compression object to use for compressing matrices in
the main graph.
Returns:
Logits.
"""
# We instantiate all variables using tf.compat.v1.get_variable() instead of
# tf.Variable() in order to share variables across multiple GPU training runs.
# If we only ran this model on a single GPU, we could simplify this function
# by replacing all instances of tf.compat.v1.get_variable() with
# tf.Variable().
#
# While instantiating conv and local layers, we add mask and threshold
# variables to the layer by calling the pruning.apply_mask() function.
# Note that the masks are applied only to the weight tensors
# conv1
with tf.variable_scope('conv1') as scope:
kernel = _variable_with_weight_decay(
'weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=0.0)
conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv1)
# pool1
pool1 = tf.nn.max_pool(
conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# norm1
norm1 = tf.nn.lrn(
pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')
# conv2
with tf.variable_scope('conv2') as scope:
kernel = _variable_with_weight_decay(
'weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=0.0)
conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv2)
# norm2
norm2 = tf.nn.lrn(
conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
# pool2
pool2 = tf.nn.max_pool(
norm2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# local3
with tf.variable_scope('local3') as scope:
# Move everything into depth so we can perform a single matrix multiply.
reshape = tf.reshape(pool2, [BATCH_SIZE, -1])
dim = reshape.get_shape()[1].value
weights = _variable_with_weight_decay(
'weights', shape=[dim, 384], stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1))
local3 = tf.nn.relu(
tf.matmul(reshape, compression_obj.apply_compression(weights, scope)) +
biases,
name=scope.name)
_activation_summary(local3)
# local4
with tf.variable_scope('local4') as scope:
weights = _variable_with_weight_decay(
'weights', shape=[384, 192], stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1))
local4 = tf.nn.relu(
tf.matmul(local3, compression_obj.apply_compression(weights, scope)) +
biases,
name=scope.name)
_activation_summary(local4)
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
with tf.variable_scope('softmax_linear') as scope:
weights = _variable_with_weight_decay(
'weights', [192, NUM_CLASSES], stddev=1 / 192.0, wd=0.0)
biases = _variable_on_cpu('biases', [NUM_CLASSES],
tf.constant_initializer(0.0))
softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)
_activation_summary(softmax_linear)
return softmax_linear
# _*_ coding utf-8 _*_
# Author：94342
# Time： 2020/9/1717:09
# File： New03.py
# Engine：PyCharm
import tensorflow.compat.v1 as tf
if __name__ == '__main__':
tf.compat.v1.disable_eager_execution()
v1 = tf.Variable(0, name='counter')
one = tf.constant(1)
temp = tf.add(v1, one)
process = tf.assign(v1, temp)
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
print(sess.run(v1))
for i in range(3):
sess.run(process)
print(sess.run(v1))
# In[1]: # Task 1 - 1 # TIES 4911 # Toni Pikkarainen # 14.1.2020 import tensorflow.compat.v1 as tf tf.disable_v2_behavior() # In[2]: # Constants # Lecture02, slides 11-13 a = tf.constant(5) b = tf.constant(2) add_op = tf.add(a, b) mul_op = tf.multiply(b, add_op) pow_op = tf.pow(mul_op, b) # In[3]: # Variables # Lecture02, slides 15-17 var1 = tf.Variable(2, name="scalar1") var2 = tf.Variable(3, name="scalar2") assign_op = var2.assign(10) # In[4]: # Placeholders # Lecture02, slides 19-20
def dilated_layer(name,
input_batch,
dilation,
dilation_width,
dilation_units,
residual=False,
dense_residual=False,
to_batch_norm=False):
'''Create a dilation layer:
INPUT:
name: must be unique for graph purpose
input_batch: 3D input tensor batch, length, channels/width
Current implementation keeps the channels/hidden units intakt
dialton: dialtion rate to apply
dilation_width: with of the dilation filter (only 2 supported?)
dilation_units: dilation_units or channels
residual: True/False --> select if to propagate residual in that layer/stack
to_batch_norm: True/False select of to perform batch norm at every layer
RETURNS:
3D tensor batch, length-dilation rate, channels width'''
with tf.name_scope(name):
# get shapes
channels = input_batch.get_shape().as_list()[2]
# create variables
dilation_weights = create_variable(
'dilation_weights', [dilation_width, channels, dilation_units])
dilation_biases = create_bias_variable('dilation_biases',
[dilation_units])
gate_weights = create_variable(
'gate_weights', [dilation_width, channels, dilation_units])
gate_biases = create_bias_variable('gate_biases', [dilation_units])
# redisual and skip
if residual == True:
if dense_residual == True:
dense_weights = create_variable(
'dense_weights',
[dilation_units, channels, dilation_units])
dense_biases = create_bias_variable('dense_biases',
[dilation_units])
# skip_weights = create_variable('skip_weights', [1, dilation_units, skip_units])
# skip_biases = create_bias_variable('skip_biases', [skip_units])
# define convolutional steps
dilated = tf.add(
dilated_conv(input_batch, dilation_weights, dilation=dilation),
dilation_biases)
gated = tf.add(
dilated_conv(input_batch, gate_weights, dilation=dilation),
gate_biases)
dilated_gated = tf.tanh(dilated) * tf.sigmoid(gated)
if residual == True:
# if dense residual connection desired make a 1x1 convolutiion before adding
if dense_residual == True:
# 1x1 dense convolution for residual
transformed = tf.nn.conv1d(dilated_gated,
dense_weights,
stride=1,
padding="SAME",
name="dense")
transformed = transformed + dense_biases
# add up residual to 1x1 transformed output
out = input_batch + transformed
else:
# else just add input_batch shortcut to dilated/gated
out = input_batch + dilated_gated
else:
# else dilated gated is out
out = dilated_gated
# # The 1x1 conv to produce the skip output
# skip_cut = tf.shape(out)[1] - output_width
# out_skip = tf.slice(out, [0, skip_cut, 0], [-1, -1, -1])
# weights_skip = variables['skip']
# skip_contribution = tf.nn.conv1d(
# out_skip, weights_skip, stride=1, padding="SAME", name="skip")
# # batch norm
if to_batch_norm == True:
out = tf.layers.batch_normalization(out)
# make summary histograms of weights
tf.summary.histogram(name + '_dilation_weights', dilation_weights)
tf.summary.histogram(name + '_dilation_biases', dilation_biases)
tf.summary.histogram(name + '_gate_weights', gate_weights)
tf.summary.histogram(name + '_gate_biases', gate_biases)
if dense_residual == True:
tf.summary.histogram(name + '_dense_weights', dense_weights)
tf.summary.histogram(name + '_dense_biases', dense_biases)
# tf.summary.histogram(name + '_skip_weights', skip_weights)
# tf.summary.histogram(name + '_skip_biases', skip_biases)
return out
def inference(seqs, conv_layers, hidden_units_scheme, kernel_width_scheme,
max_pool_scheme, dilation_scheme, dilation_units, dilation_width,
dilation_residual, dilation_residual_dense, dilation_batch_norm,
num_classes, batch_size, keep_prob_inner, keep_prob_outer,
seed_weights, seed_scheme, seed_weights_list,
report_conv_hidden_state):
"""INFERENCE
Args:
Returns:
regression score or hidden representation after convolutional but before dilated convolutional layer
"""
print('seqs shape')
print(seqs.get_shape().as_list())
current_layer = seqs
# Convolutional Stack with Max Pooling =====================================
# run an inital dilated layer with dilation 1 to map to the dilational unit output
with tf.name_scope('Convolutional_stack'):
for i in range(conv_layers):
j = i + 1
k = i * 2
if seed_weights and seed_scheme[i] == 1:
weights_load_string = 'arr_' + str(k)
biases_load_string = 'arr_' + str(k + 1)
print('Pre-seeding Layer: ' + str(j))
current_layer = convolutional_layer(
'conv_layer{}'.format(j),
current_layer,
hidden_units_scheme[i],
kernel_width_scheme[i],
max_pool_scheme[i],
keep_prob_inner,
True,
seed_weights_list[weights_load_string],
seed_weights_list[biases_load_string],
to_batch_norm=False)
else:
current_layer = convolutional_layer('conv_layer{}'.format(j),
current_layer,
hidden_units_scheme[i],
kernel_width_scheme[i],
max_pool_scheme[i],
keep_prob_inner,
False,
"dummy",
"dummy",
to_batch_norm=False)
print('Conv %s shape' % j)
print(current_layer.get_shape().as_list())
# Report only HIdden STate afte r Conv layers (optional) ===================
if report_conv_hidden_state:
''''only report hte hidden state after the convolutional stacks'''
hidden_conv_state = current_layer
return (hidden_conv_state)
# Dilational Layers stack ==================================================
# run an inital dilated layer with dilation 1 to map to the dilational unit output
with tf.name_scope('dilated_stack'):
current_layer = dilated_layer('dilated_layer1',
current_layer,
1,
dilation_width,
dilation_units,
residual=dilation_residual,
dense_residual=dilation_residual_dense,
to_batch_norm=dilation_batch_norm)
print('Dilated shape')
print(current_layer.get_shape().as_list())
for i, dilation in enumerate(dilation_scheme):
i = i + 1 # skipping 0 count as this is pre-established
current_layer = dilated_layer(
'dilated_layer{}'.format(i),
current_layer,
dilation,
dilation_width,
dilation_units,
residual=dilation_residual,
dense_residual=dilation_residual_dense,
to_batch_norm=dilation_batch_norm)
print('Dilated shape')
print(current_layer.get_shape().as_list())
# reshape for FC layer
with tf.name_scope('reshape_layer'):
fully_connected_width = current_layer.get_shape().as_list(
)[1] * dilation_units
current_layer = tf.reshape(current_layer,
[batch_size, fully_connected_width])
print('fully connection reshaped')
print(current_layer.get_shape().as_list())
# Final full connection(s) into logits
with tf.name_scope('final_dense'):
weights = create_variable('weights',
[fully_connected_width, num_classes])
biases = create_bias_variable('biases', [num_classes])
regression_score = tf.add(tf.matmul(current_layer, weights), biases)
print('Regression score shape')
print(regression_score.get_shape().as_list())
_activation_summary(regression_score)
tf.summary.histogram('final_dense_weights', weights)
# return regression_score, into_dilation
return regression_score
def _assemble_graph(self):
self._log('Parsing dataset...')
self._graph_parse_data()
self._log('Creating layer parameters...')
self._add_layers_to_graph()
self._log('Assembling graph...')
x, y = tf.train.shuffle_batch([self._train_images, self._train_labels],
batch_size=self._batch_size,
num_threads=self._num_threads,
capacity=self._queue_capacity,
min_after_dequeue=self._batch_size)
# Reshape input to the expected image dimensions
x = tf.reshape(x,
shape=[
-1, self._image_height, self._image_width,
self._image_depth
])
# Run the network operations
xx = self.forward_pass(x, deterministic=False)
# Define regularization cost
if self._reg_coeff is not None:
l2_cost = tf.squeeze(
tf.reduce_sum([
layer.regularization_coefficient *
tf.nn.l2_loss(layer.weights) for layer in self._layers
if isinstance(layer, layers.fullyConnectedLayer)
]))
else:
l2_cost = 0.0
# Define cost function
if self._loss_fn == 'l1':
l1_loss = tf.reduce_mean(tf.abs(tf.subtract(xx, y)))
gt = tf.reduce_sum(y, axis=[1, 2, 3]) / (32**2.0)
pr = tf.reduce_sum(xx, axis=[1, 2, 3]) / (32**2.0)
accuracy = tf.reduce_mean(tf.abs(gt - pr))
self._graph_ops['cost'] = tf.squeeze(tf.add(l1_loss, l2_cost))
self._graph_ops['accuracy'] = accuracy
# Set the optimizer and get the gradients from it
gradients, variables, global_grad_norm = self._graph_add_optimizer()
# Calculate validation and test accuracy
if self._testing:
x_test, self._graph_ops['y_test'] = tf.train.batch(
[self._test_images, self._test_labels],
batch_size=self._batch_size,
num_threads=self._num_threads,
capacity=self._queue_capacity)
x_test = tf.reshape(x_test,
shape=[
-1, self._image_height, self._image_width,
self._image_depth
])
if self._validation:
x_val, self._graph_ops['y_val'] = tf.train.batch(
[self._val_images, self._val_labels],
batch_size=self._batch_size,
num_threads=self._num_threads,
capacity=self._queue_capacity)
x_val = tf.reshape(x_val,
shape=[
-1, self._image_height, self._image_width,
self._image_depth
])
if self._testing:
self._graph_ops['x_test_predicted'] = self.forward_pass(
x_test, deterministic=True)
if self._validation:
self._graph_ops['x_val_predicted'] = self.forward_pass(
x_val, deterministic=True)
if self._testing:
if self._loss_fn == 'l1':
self._graph_ops['test_losses'] = tf.reduce_mean(
tf.abs(
tf.subtract(self._graph_ops['y_test'],
self._graph_ops['x_test_predicted'])))
gt_test = tf.reduce_sum(self._graph_ops['y_test'],
axis=[1, 2, 3]) / (32**2.0)
pr_test = tf.reduce_sum(self._graph_ops['x_test_predicted'],
axis=[1, 2, 3]) / (32**2.0)
self._graph_ops['gt_test'] = gt_test
self._graph_ops['pr_test'] = pr_test
self._graph_ops['test_accuracy'] = tf.reduce_mean(
tf.abs(gt_test - pr_test))
if self._validation:
if self._loss_fn == 'l1':
self._graph_ops['val_losses'] = tf.reduce_mean(
tf.abs(
tf.subtract(self._graph_ops['y_val'],
self._graph_ops['x_val_predicted'])))
gt_val = tf.reduce_sum(self._graph_ops['y_val'],
axis=[1, 2, 3]) / (32**2.0)
pr_val = tf.reduce_sum(self._graph_ops['x_val_predicted'],
axis=[1, 2, 3]) / (32**2.0)
self._graph_ops['val_accuracy'] = tf.reduce_mean(
tf.abs(gt_val - pr_val))
# Epoch summaries for Tensorboard
if self._tb_dir is not None:
self._graph_tensorboard_summary(l2_cost, gradients, variables,
global_grad_norm)
def compute_gradients(self,
loss,
var_list,
gate_gradients=GATE_OP,
aggregation_method=None,
colocate_gradients_with_ops=False,
grad_loss=None,
gradient_tape=None):
if callable(loss):
# TF is running in Eager mode, check we received a vanilla tape.
if not gradient_tape:
raise ValueError('When in Eager mode, a tape needs to be passed.')
vector_loss = loss()
if self._num_microbatches is None:
self._num_microbatches = tf.shape(input=vector_loss)[0]
sample_state = self._dp_sum_query.initial_sample_state(var_list)
microbatches_losses = tf.reshape(vector_loss,
[self._num_microbatches, -1])
sample_params = (
self._dp_sum_query.derive_sample_params(self._global_state))
def process_microbatch(i, sample_state):
"""Process one microbatch (record) with privacy helper."""
microbatch_loss = tf.reduce_mean(
input_tensor=tf.gather(microbatches_losses, [i]))
grads = gradient_tape.gradient(microbatch_loss, var_list)
sample_state = self._dp_sum_query.accumulate_record(
sample_params, sample_state, grads)
return sample_state
for idx in range(self._num_microbatches):
sample_state = process_microbatch(idx, sample_state)
grad_sums, self._global_state = (
self._dp_sum_query.get_noised_result(
sample_state, self._global_state))
def normalize(v):
return v / tf.cast(self._num_microbatches, tf.float32)
final_grads = tf.nest.map_structure(normalize, grad_sums)
grads_and_vars = list(zip(final_grads, var_list))
return grads_and_vars
else:
# TF is running in graph mode, check we did not receive a gradient tape.
if gradient_tape:
raise ValueError('When in graph mode, a tape should not be passed.')
# Note: it would be closer to the correct i.i.d. sampling of records if
# we sampled each microbatch from the appropriate binomial distribution,
# although that still wouldn't be quite correct because it would be
# sampling from the dataset without replacement.
if self._num_microbatches is None:
self._num_microbatches = tf.shape(input=loss)[0]
microbatches_losses = tf.reshape(loss, [self._num_microbatches, -1])
sample_params = (
self._dp_sum_query.derive_sample_params(self._global_state))
def process_microbatch(i, sample_state):
"""Process one microbatch (record) with privacy helper."""
grads, _ = zip(*super(cls, self).compute_gradients(
tf.reduce_mean(input_tensor=tf.gather(
microbatches_losses, [i])), var_list, gate_gradients,
aggregation_method, colocate_gradients_with_ops, grad_loss))
grads_list = [
g if g is not None else tf.zeros_like(v)
for (g, v) in zip(list(grads), var_list)
]
sample_state = self._dp_sum_query.accumulate_record(
sample_params, sample_state, grads_list)
return sample_state
if var_list is None:
var_list = (
tf.compat.v1.trainable_variables() + tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
sample_state = self._dp_sum_query.initial_sample_state(var_list)
if self._unroll_microbatches:
for idx in range(self._num_microbatches):
sample_state = process_microbatch(idx, sample_state)
else:
# Use of while_loop here requires that sample_state be a nested
# structure of tensors. In general, we would prefer to allow it to be
# an arbitrary opaque type.
cond_fn = lambda i, _: tf.less(i, self._num_microbatches)
body_fn = lambda i, state: [tf.add(i, 1), process_microbatch(i, state)] # pylint: disable=line-too-long
idx = tf.constant(0)
_, sample_state = tf.while_loop(
cond=cond_fn, body=body_fn, loop_vars=[idx, sample_state])
grad_sums, self._global_state = (
self._dp_sum_query.get_noised_result(
sample_state, self._global_state))
def normalize(v):
return tf.truediv(v, tf.cast(self._num_microbatches, tf.float32))
final_grads = tf.nest.map_structure(normalize, grad_sums)
return list(zip(final_grads, var_list))
potentials = []
valid_potentials = []
wave_functions = []
valid_functions = []
sess = tf.Session()
sess.run(init)
for i in range(npots):
if i % 10 == 0:
print(str((100. * i) / npots) + '% complete')
for j in range(3):
vofx = generate_potential(j, (1. * i) / npots)
energy = tf.reduce_mean(
tf.subtract(
tf.multiply(tf.square(psi), tf.add(vofx, 1. * bins * bins)),
tf.multiply(tf.multiply(tf.add(psil, psir), psi),
0.5 * bins * bins)))
training = optim.minimize(energy)
sess.run(reinit)
for t in range(20000):
sess.run(training)
sess.run(renorm)
if i % validnth == 0:
valid_potentials.append(vofx)
valid_functions.append(sess.run(psi).tolist())
else:
potentials.append(vofx)
wave_functions.append(sess.run(psi).tolist())
with open('test_potentials' + str(seed) + '.csv', 'w') as f:
def convolutional_block(X_input,
kernel_size,
in_filter,
out_filters,
stage,
block,
training,
stride=2):
"""
Implementation of the convolutional block as defined in Figure 4
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
training -- train or test
stride -- Integer, specifying the stride to be used
Returns:
X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
block_name = 'res' + str(stage) + block
with tf.variable_scope(block_name):
f1, f2, f3 = out_filters
x_shortcut = X_input
#first
W_conv1 = weight_variable([1, 1, in_filter, f1])
X = tf.nn.conv2d(X_input,
W_conv1,
strides=[1, stride, stride, 1],
padding='VALID')
X = tf.layers.batch_normalization(X, axis=3, training=training)
X = tf.nn.relu(X)
#second
W_conv2 = weight_variable([kernel_size, kernel_size, f1, f2])
X = tf.nn.conv2d(X, W_conv2, strides=[1, 1, 1, 1], padding='SAME')
X = tf.layers.batch_normalization(X, axis=3, training=training)
X = tf.nn.relu(X)
#third
W_conv3 = weight_variable([1, 1, f2, f3])
X = tf.nn.conv2d(X, W_conv3, strides=[1, 1, 1, 1], padding='VALID')
X = tf.layers.batch_normalization(X, axis=3, training=training)
#shortcut path
W_shortcut = weight_variable([1, 1, in_filter, f3])
x_shortcut = tf.nn.conv2d(x_shortcut,
W_shortcut,
strides=[1, stride, stride, 1],
padding='VALID')
#final
add = tf.add(x_shortcut, X)
add_result = tf.nn.relu(add)
return add_result
def prepare_processing_graph(self, model_settings):
"""Builds a TensorFlow graph to apply the input distortions.
Creates a graph that loads a WAVE file, decodes it, scales the volume,
shifts it in time, adds in background noise, calculates a spectrogram, and
then builds an MFCC fingerprint from that.
This must be called with an active TensorFlow session running, and it
creates multiple placeholder inputs, and one output:
- wav_filename_placeholder_: Filename of the WAV to load.
- foreground_volume_placeholder_: How loud the main clip should be.
- time_shift_padding_placeholder_: Where to pad the clip.
- time_shift_offset_placeholder_: How much to move the clip in time.
- background_data_placeholder_: PCM sample data for background noise.
- background_volume_placeholder_: Loudness of mixed-in background.
- mfcc_: Output 2D fingerprint of processed audio.
Args:
model_settings: Information about the current model being trained.
"""
desired_samples = model_settings['desired_samples']
channel_count = model_settings['channel_count']
sample_rate = model_settings['sample_rate']
self.foreground_data_placeholder_ = tf.placeholder(
tf.float32, [desired_samples, channel_count])
# Allow the audio sample's volume to be adjusted.
self.foreground_volume_placeholder_ = tf.placeholder(tf.float32, [])
scaled_foreground = tf.multiply(self.foreground_data_placeholder_,
self.foreground_volume_placeholder_)
# Shift the sample's start position, and pad any gaps with zeros.
self.time_shift_padding_placeholder_ = tf.placeholder(tf.int32, [2, 2])
self.time_shift_offset_placeholder_ = tf.placeholder(tf.int32, [2])
padded_foreground = tf.pad(scaled_foreground,
self.time_shift_padding_placeholder_,
mode='CONSTANT')
sliced_foreground = tf.slice(padded_foreground,
self.time_shift_offset_placeholder_,
[desired_samples, -1])
# Mix in background noise.
self.background_data_placeholder_ = tf.placeholder(
tf.float32, [desired_samples, channel_count])
self.background_volume_placeholder_ = tf.placeholder(tf.float32, [])
background_mul = tf.multiply(self.background_data_placeholder_,
self.background_volume_placeholder_)
background_add = tf.add(background_mul, sliced_foreground)
background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)
# Run the spectrogram and MFCC ops to get a 2D 'fingerprint' of the audio.
self.waveform_ = background_clamp
spectrograms = []
for ichannel in range(channel_count):
spectrograms.append(
audio_ops.audio_spectrogram(
tf.slice(background_clamp, [0, ichannel], [-1, 1]),
window_size=model_settings['window_size_samples'],
stride=model_settings['window_stride_samples'],
magnitude_squared=True))
self.spectrogram_ = tf.stack(spectrograms, -1)
mfccs = []
for ichannel in range(channel_count):
mfccs.append(
audio_ops.mfcc(
spectrograms[ichannel],
sample_rate,
upper_frequency_limit=model_settings['sample_rate'] // 2,
filterbank_channel_count=model_settings[
'filterbank_channel_count'],
dct_coefficient_count=model_settings[
'dct_coefficient_count']))
self.mfcc_ = tf.stack(mfccs, -1)