def discriminator(self, image, y=None, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse:
scope.reuse_variables()
if not self.y_dim:
h0 = lrelu(conv2d(image, self.df_dim, name='d_h0_conv'))
h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim*2, name='d_h1_conv')))
h2 = lrelu(self.d_bn2(conv2d(h1, self.df_dim*4, name='d_h2_conv')))
h3 = lrelu(self.d_bn3(conv2d(h2, self.df_dim*8, name='d_h3_conv')))
h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4), h4
else:
yb = tf.reshape(y, [self.batch_size, 1, 1, self.y_dim])
x = conv_cond_concat(image, yb)
h0 = lrelu(conv2d(x, self.c_dim + self.y_dim, name='d_h0_conv'))
h0 = conv_cond_concat(h0, yb)
h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim + self.y_dim, name='d_h1_conv')))
h1 = tf.reshape(h1, [self.batch_size, -1])
h1 = tf.concat_v2([h1, y], 1)
h2 = lrelu(self.d_bn2(linear(h1, self.dfc_dim, 'd_h2_lin')))
h2 = tf.concat_v2([h2, y], 1)
h3 = linear(h2, 1, 'd_h3_lin')
return tf.nn.sigmoid(h3), h3
def _testConfMatrixOnTensors(self, tf_dtype, np_dtype):
with self.test_session() as sess:
m_neg = tf.placeholder(dtype=tf.float32)
m_pos = tf.placeholder(dtype=tf.float32)
s = tf.placeholder(dtype=tf.float32)
neg = tf.random_normal([20], mean=m_neg, stddev=s, dtype=tf.float32)
pos = tf.random_normal([20], mean=m_pos, stddev=s, dtype=tf.float32)
data = tf.concat_v2([neg, pos], 0)
data = tf.cast(tf.round(data), tf_dtype)
data = tf.minimum(tf.maximum(data, 0), 1)
lab = tf.concat_v2(
[tf.zeros(
[20], dtype=tf_dtype), tf.ones(
[20], dtype=tf_dtype)], 0)
cm = tf.confusion_matrix(
lab, data, dtype=tf_dtype, num_classes=2)
d, l, cm_out = sess.run([data, lab, cm], {m_neg: 0.0,
m_pos: 1.0,
s: 1.0})
truth = np.zeros([2, 2], dtype=np_dtype)
try:
range_builder = xrange
except NameError: # In Python 3.
range_builder = range
for i in range_builder(len(d)):
truth[d[i], l[i]] += 1
self.assertEqual(cm_out.dtype, np_dtype)
self.assertAllClose(cm_out, truth, atol=1e-10)
def __call__(self, input_tuple, state, scope=None):
with vs.variable_scope(scope or type(self).__name__):
self._init_parameters()
_input, target, dInput_dF = input_tuple
u, s, r, inner_spikes, dW, dF, reward, reward_mean = state
s = (1.0 - tau_syn) * s + tf.concat_v2([_input, inner_spikes], 1)
u = (1.0 - tau_mem) * u + mo.matmul(s, self.W)
r = (1.0 - tau_refr) * r
act_raw = self._activation(u)
act = act_raw * tf.exp(-r)
spikes = tf.where(
act > tf.random_uniform([batch_size, self._num_units]),
tf.ones([batch_size, self._num_units]),
tf.zeros([batch_size, self._num_units])
)
hidden_spikes, _ = self.slice(spikes)
_, act_visible = self.slice(act)
reward_mean = (1.0 - tau_long) * reward_mean + tau_long * reward
reward = (1.0 - tau_learn) * reward + tau_learn * tf.reduce_sum(
target * safe_log(act_visible*dt) + (1.0 - target) * safe_log(1.0 - act_visible*dt)
, 1)
r += spikes * amp_refr
act_grad = tf.gradients([act], [u])[0]
learn_target = tf.concat_v2([hidden_spikes, target], 1)
factor = tf.concat_v2([
(reward - reward_mean) * tf.ones((batch_size, hidden_size,)),
tf.ones((batch_size, visible_size,))
], 1)
neuron_derivative = tf.reduce_sum( (act_grad/act_raw) * (learn_target - act) * factor, 0)
Wsliced = tf.slice(self.W, [0, 0], [self._filters_num, self._num_units])
dF_deriv_part = tf.squeeze(mo.matmul(Wsliced, tf.expand_dims(neuron_derivative, 1)))
dW += lrate * outer(
tf.reduce_sum(s, 0),
neuron_derivative
)
dInput_dF = tf.reduce_mean(dInput_dF, 0)
dF += lrate * dF_deriv_part * dInput_dF
return GLMOutputTuple(spikes, act, factor, reward, reward_mean), GLMStateTuple(u, s, r, spikes, dW, dF, reward, reward_mean)
def testAttentionCellWrapperCorrectResult(self):
num_units = 4
attn_length = 6
batch_size = 2
expected_output = np.array(
[[0.955392, 0.408507, -0.60122, 0.270718],
[0.903681, 0.331165, -0.500238, 0.224052]],
dtype=np.float32)
expected_state = np.array(
[[
0.81331915, 0.32036272, 0.28079176, 1.08888793, 0.41264394,
0.1062041, 0.10444493, 0.32050529, 0.64655536, 0.70794445,
0.51896095, 0.31809306, 0.58086717, 0.49446869, 0.7641536,
0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
0.99211812, 0.12295902, 1.01412082, 0.33123279, -0.71114945,
0.40583119
], [
0.59962207, 0.42597458, -0.22491696, 0.98063421, 0.32548007,
0.11623692, -0.10100613, 0.27708149, 0.76956916, 0.6360054,
0.51719815, 0.50458527, 0.73000264, 0.66986895, 0.73576689,
0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
0.36127412, 0.12125921, 0.99780077, 0.31886846, -0.67595094,
0.56531656
]],
dtype=np.float32)
seed = 12345
tf.set_random_seed(seed)
for state_is_tuple in [False, True]:
with tf.Session() as sess:
with tf.variable_scope("state_is_tuple", reuse=state_is_tuple):
lstm_cell = tf.contrib.rnn.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = tf.contrib.rnn.AttentionCellWrapper(
lstm_cell, attn_length, state_is_tuple=state_is_tuple)
zeros1 = tf.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 1)
zeros2 = tf.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 2)
zeros3 = tf.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 3)
attn_state_zeros = tf.random_uniform(
(batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4)
zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
if not state_is_tuple:
zero_state = tf.concat_v2([
zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2]
], 1)
inputs = tf.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 5)
output, state = cell(inputs, zero_state)
if state_is_tuple:
state = tf.concat_v2([state[0][0], state[0][1], state[1], state[2]],
1)
sess.run(tf.global_variables_initializer())
self.assertAllClose(sess.run(output), expected_output)
self.assertAllClose(sess.run(state), expected_state)
def _define_partial_maximization_operation(self, shard_id, shard):
"""Computes the partial statistics of the means and covariances.
Args:
shard_id: current shard id.
shard: current data shard, 1 X num_examples X dimensions.
"""
# Soft assignment of each data point to each of the two clusters.
self._points_in_k[shard_id] = tf.reduce_sum(self._w[shard_id],
0,
keep_dims=True)
# Partial means.
w_mul_x = tf.expand_dims(
tf.matmul(self._w[shard_id],
tf.squeeze(shard, [0]),
transpose_a=True), 1)
self._w_mul_x.append(w_mul_x)
# Partial covariances.
x = tf.concat_v2([shard for _ in range(self._num_classes)], 0)
x_trans = tf.transpose(x, perm=[0, 2, 1])
x_mul_w = tf.concat_v2([
tf.expand_dims(x_trans[k, :, :] * self._w[shard_id][:, k], 0)
for k in range(self._num_classes)
], 0)
self._w_mul_x2.append(tf.matmul(x_mul_w, x))
def predict_mean_and_var(self, Fmu, Fvar):
mu_list, var_list = zip(*[
lik.predict_mean_and_var(Fmu, Fvar) for lik in self.likelihood_list
])
mu = tf.concat_v2(mu_list, 1)
var = tf.concat_v2(var_list, 1)
return mu, var
def _define_distance_to_clusters(self, data):
"""Defines the Mahalanobis distance to the assigned Gaussian."""
# TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
# mean) from log probability function.
self._all_scores = []
for shard in data:
all_scores = []
shard = tf.expand_dims(shard, 0)
for c in xrange(self._num_classes):
if self._covariance_type == FULL_COVARIANCE:
cov = self._covs[c, :, :]
elif self._covariance_type == DIAG_COVARIANCE:
cov = tf.diag(self._covs[c, :])
inverse = tf.matrix_inverse(cov + self._min_var)
inv_cov = tf.tile(
tf.expand_dims(inverse, 0), tf.stack([self._num_examples, 1, 1]))
diff = tf.transpose(shard - self._means[c, :, :], perm=[1, 0, 2])
m_left = tf.matmul(diff, inv_cov)
all_scores.append(
tf.sqrt(tf.matmul(
m_left, tf.transpose(
diff, perm=[0, 2, 1]))))
self._all_scores.append(
tf.reshape(
tf.concat_v2(all_scores, 1),
tf.stack([self._num_examples, self._num_classes])))
# Distance to the associated class.
self._all_scores = tf.concat_v2(self._all_scores, 0)
assignments = tf.concat_v2(self.assignments(), 0)
rows = tf.to_int64(tf.range(0, self._num_examples))
indices = tf.concat_v2(
[tf.expand_dims(rows, 1), tf.expand_dims(assignments, 1)], 1)
self._scores = tf.gather_nd(self._all_scores, indices)
def partRotator(self, images, name, batch_size=10, reuse=False):
#HW 40x40, 32x40, 32x48
with tf.variable_scope(name) as scope:
if reuse:
scope.reuse_variables()
c0 = lrelu(conv2d(images, self.gf_dim, d_h=1, d_w=1, name="p_conv0"))
c0r = resblock(c0, name="p_conv0_res")
c1 = lrelu(batch_norm(conv2d(c0r, self.gf_dim*2, name="p_conv1"),name="p_bnc1"))
#down1
c1r = resblock(c1, name="p_conv1_res")
c2 = lrelu(batch_norm(conv2d(c1r, self.gf_dim*4, name='p_conv2'),name="p_bnc2"))
#down2
c2r = resblock(c2, name="p_conv2_res")
c3 = lrelu(batch_norm(conv2d(c2r, self.gf_dim*8, name='p_conv3'),name="p_bnc3"))
#down3 5x5, 4x5, 4x6
c3r = resblock(c3, name="p_conv3_res")
c3r2 = resblock(c3r, name="p_conv3_res2")
shape = c3r2.get_shape().as_list()
d1 = lrelu(batch_norm(deconv2d(c3r2, [shape[0], shape[1] * 2, shape[2] * 2, self.gf_dim*4], name="p_deconv1"), name="p_bnd1"))
#up1
after_select_d1 = lrelu(batch_norm(conv2d(tf.concat_v2([d1, c2r], axis=3), self.gf_dim*4, d_h=1, d_w=1, name="p_deconv1_s"),name="p_bnd1_s"))
d1_r = resblock(after_select_d1, name="p_deconv1_res")
d2 = lrelu(batch_norm(deconv2d(d1_r, [shape[0], shape[1] * 4, shape[2] * 4, self.gf_dim*2], name="p_deconv2"), name="p_bnd2"))
#up2
after_select_d2 = lrelu(batch_norm(conv2d(tf.concat_v2([d2, c1r], axis=3), self.gf_dim*2, d_h=1, d_w=1, name="p_deconv2_s"),name="p_bnd2_s"))
d2_r = resblock(after_select_d2, name="p_deconv2_res")
d3 = lrelu(batch_norm(deconv2d(d2_r, [shape[0], shape[1] * 8, shape[2] * 8, self.gf_dim], name="p_deconv3"), name="p_bnd3"))
#up3
after_select_d3 = lrelu(batch_norm(conv2d(tf.concat_v2([d3, c0r], axis=3), self.gf_dim, d_h=1, d_w=1, name="p_deconv3_s"),name="p_bnd3_s"))
d3_r = resblock(after_select_d3, name="p_deconv3_res")
check_part = tf.nn.tanh(conv2d(d3_r, 3, d_h=1, d_w=1, name="p_check"))
return d3_r, check_part
def _define_distance_to_clusters(self, data):
"""Defines the Mahalanobis distance to the assigned Gaussian."""
# TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
# mean) from log probability function.
self._all_scores = []
for shard in data:
all_scores = []
shard = tf.expand_dims(shard, 0)
for c in xrange(self._num_classes):
if self._covariance_type == FULL_COVARIANCE:
cov = self._covs[c, :, :]
elif self._covariance_type == DIAG_COVARIANCE:
cov = tf.diag(self._covs[c, :])
inverse = tf.matrix_inverse(cov + self._min_var)
inv_cov = tf.tile(tf.expand_dims(inverse, 0),
tf.stack([self._num_examples, 1, 1]))
diff = tf.transpose(shard - self._means[c, :, :],
perm=[1, 0, 2])
m_left = tf.matmul(diff, inv_cov)
all_scores.append(
tf.sqrt(
tf.matmul(m_left, tf.transpose(diff, perm=[0, 2, 1]))))
self._all_scores.append(
tf.reshape(tf.concat_v2(all_scores, 1),
tf.stack([self._num_examples, self._num_classes])))
# Distance to the associated class.
self._all_scores = tf.concat_v2(self._all_scores, 0)
assignments = tf.concat_v2(self.assignments(), 0)
rows = tf.to_int64(tf.range(0, self._num_examples))
indices = tf.concat_v2(
[tf.expand_dims(rows, 1),
tf.expand_dims(assignments, 1)], 1)
self._scores = tf.gather_nd(self._all_scores, indices)
def symmetric_feedforward_weights(weights): sizes = [weights[0].get_shape()[0].value] sizes += [ w.get_shape()[0].value for w in weights[1:] ] sizes += [weights[-1].get_shape()[1].value] res = [] for pre_id, pre_size in enumerate(sizes): stack = [] for post_id, post_size in enumerate(sizes): if pre_id == post_id - 1 and pre_id < len(weights): stack.append(weights[pre_id]) elif post_id == pre_id -1 and post_id < len(weights): stack.append(tf.transpose(weights[post_id])) else: pre_zeros = tf.zeros((pre_size, post_size)) stack.append(pre_zeros) res.append(tf.concat_v2(stack, 1)) res = tf.concat_v2(res, 0) # for i in xrange(res_v.shape[0]): # for j in xrange(res_v.shape[1]): # assert res_v[i, j] == res_v[j, i] return res
def symmetric_feedforward_weights(weights):
sizes = [weights[0].get_shape()[0].value]
sizes += [w.get_shape()[0].value for w in weights[1:]]
sizes += [weights[-1].get_shape()[1].value]
res = []
for pre_id, pre_size in enumerate(sizes):
stack = []
for post_id, post_size in enumerate(sizes):
if pre_id == post_id - 1 and pre_id < len(weights):
stack.append(weights[pre_id])
elif post_id == pre_id - 1 and post_id < len(weights):
stack.append(tf.transpose(weights[post_id]))
else:
pre_zeros = tf.zeros((pre_size, post_size))
stack.append(pre_zeros)
res.append(tf.concat_v2(stack, 1))
res = tf.concat_v2(res, 0)
# for i in xrange(res_v.shape[0]):
# for j in xrange(res_v.shape[1]):
# assert res_v[i, j] == res_v[j, i]
return res
def calcLoss(targetss, logitss, batch_sizee, num_sampless, num_stepss,
indices):
indices2 = tf.zeros([num_sampless], tf.int32)
print("indices2")
print(indices2)
#heightt=tf.to_int32(tf.shape(logitss)[0]);
heightt = logitss.get_shape().as_list()[0]
print("heightt")
print(heightt)
for i in range(1, heightt):
#print("Loop Beginsssss");
tempp = tf.fill([num_sampless], i)
indices2 = tf.concat_v2([indices2, tempp], 0)
print("indices2")
print(indices2)
indices = indices2 + tf.reshape(indices, [-1])
print("indices")
print(indices)
negative_logitss = tf.gather(tf.reshape(logitss, [-1]), indices)
negative_logitss = tf.reshape(negative_logitss,
[heightt, num_sampless])
print("logitss")
print(logitss)
print("negative_logitss")
print(negative_logitss)
print("indices")
print(indices)
print("targetss")
print(targetss)
targetss_flattened = tf.range(
0,
tf.shape(logitss)[0]) * tf.shape(logitss)[1] + targetss
print("targetss_flattened")
print(targetss_flattened)
true_logitss = tf.gather(tf.reshape(logitss, [-1]),
targetss_flattened)
print("true_logitss")
print(true_logitss)
new_targetss = tf.zeros([logitss.get_shape().as_list()[0]],
tf.int32)
print("new_targetss")
print(new_targetss)
true_logitss_2d = tf.reshape(true_logitss,
[tf.shape(true_logitss)[0], 1])
print("true_logitss_2d")
print(true_logitss_2d)
print("new_logitss")
new_logitss = tf.concat_v2([true_logitss_2d, negative_logitss], 1)
print(new_logitss)
loss = tf.nn.seq2seq.sequence_loss_by_example(
[new_logitss], [new_targetss],
[tf.ones([batch_sizee * num_stepss], dtype=data_type())])
return loss
def boston_eval_fn():
boston = tf.contrib.learn.datasets.load_boston()
n_examples = len(boston.target)
features = tf.reshape(
tf.constant(boston.data), [n_examples, _BOSTON_INPUT_DIM])
labels = tf.reshape(tf.constant(boston.target), [n_examples, 1])
return tf.concat_v2([features, features], 0), tf.concat_v2([labels, labels],
0)
def boston_eval_fn():
boston = tf.contrib.learn.datasets.load_boston()
n_examples = len(boston.target)
features = tf.reshape(tf.constant(boston.data),
[n_examples, _BOSTON_INPUT_DIM])
labels = tf.reshape(tf.constant(boston.target), [n_examples, 1])
return tf.concat_v2([features, features],
0), tf.concat_v2([labels, labels], 0)
def sampler(self, z, y=None):
with tf.variable_scope("generator") as scope:
scope.reuse_variables()
if not self.y_dim:
s_h, s_w = self.output_height, self.output_width
s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)
# project `z` and reshape
h0 = tf.reshape(
linear(z, self.gf_dim*8*s_h16*s_w16, 'g_h0_lin'),
[-1, s_h16, s_w16, self.gf_dim * 8])
h0 = tf.nn.relu(self.g_bn0(h0, train=False))
h1 = deconv2d(h0, [self.batch_size, s_h8, s_w8, self.gf_dim*4], name='g_h1')
h1 = tf.nn.relu(self.g_bn1(h1, train=False))
h2 = deconv2d(h1, [self.batch_size, s_h4, s_w4, self.gf_dim*2], name='g_h2')
h2 = tf.nn.relu(self.g_bn2(h2, train=False))
h3 = deconv2d(h2, [self.batch_size, s_h2, s_w2, self.gf_dim*1], name='g_h3')
h3 = tf.nn.relu(self.g_bn3(h3, train=False))
h4 = deconv2d(h3, [self.batch_size, s_h, s_w, self.c_dim], name='g_h4')
return tf.nn.tanh(h4)
else:
s_h, s_w = self.output_height, self.output_width
s_h2, s_h4 = int(s_h/2), int(s_h/4)
s_w2, s_w4 = int(s_w/2), int(s_w/4)
# yb = tf.reshape(y, [-1, 1, 1, self.y_dim])
yb = tf.reshape(y, [self.batch_size, 1, 1, self.y_dim])
try:
z = tf.concat_v2([z, y], 1)
catch:
z = tf.concat([z, y], 1)
h0 = tf.nn.relu(self.g_bn0(linear(z, self.gfc_dim, 'g_h0_lin')))
try:
h0 = tf.concat_v2([h0, y], 1)
catch:
h0 = tf.concat([h0, y], 1)
h1 = tf.nn.relu(self.g_bn1(
linear(h0, self.gf_dim*2*s_h4*s_w4, 'g_h1_lin'), train=False))
h1 = tf.reshape(h1, [self.batch_size, s_h4, s_w4, self.gf_dim * 2])
h1 = conv_cond_concat(h1, yb)
h2 = tf.nn.relu(self.g_bn2(
deconv2d(h1, [self.batch_size, s_h2, s_w2, self.gf_dim * 2], name='g_h2'), train=False))
h2 = conv_cond_concat(h2, yb)
return tf.nn.sigmoid(deconv2d(h2, [self.batch_size, s_h, s_w, self.c_dim], name='g_h3'))
def VI_Block(X, S1, S2, config):
k = config.k # Number of value iterations performed
ch_i = config.ch_i # Channels in input layer
ch_h = config.ch_h # Channels in initial hidden layer
ch_q = config.ch_q # Channels in q layer (~actions)
state_batch_size = config.statebatchsize # k+1 state inputs for each channel
bias = tf.Variable(np.random.randn(1, 1, 1, ch_h) * 0.01, dtype=tf.float32)
# weights from inputs to q layer (~reward in Bellman equation)
w0 = tf.Variable(np.random.randn(3, 3, ch_i, ch_h) * 0.01,
dtype=tf.float32)
w1 = tf.Variable(np.random.randn(1, 1, ch_h, 1) * 0.01, dtype=tf.float32)
w = tf.Variable(np.random.randn(3, 3, 1, ch_q) * 0.01, dtype=tf.float32)
# feedback weights from v layer into q layer (~transition probabilities in Bellman equation)
w_fb = tf.Variable(np.random.randn(3, 3, 1, ch_q) * 0.01, dtype=tf.float32)
w_o = tf.Variable(np.random.randn(ch_q, 8) * 0.01, dtype=tf.float32)
# initial conv layer over image+reward prior
h = conv2d_flipkernel(X, w0, name="h0") + bias
r = conv2d_flipkernel(h, w1, name="r")
q = conv2d_flipkernel(r, w, name="q")
v = tf.reduce_max(q, axis=3, keep_dims=True, name="v")
for i in range(0, k - 1):
rv = tf.concat_v2([r, v], 3)
wwfb = tf.concat_v2([w, w_fb], 2)
q = conv2d_flipkernel(rv, wwfb, name="q")
v = tf.reduce_max(q, axis=3, keep_dims=True, name="v")
# do one last convolution
q = conv2d_flipkernel(tf.concat_v2([r, v], 3),
tf.concat_v2([w, w_fb], 2),
name="q")
# CHANGE TO THEANO ORDERING
# Since we are selecting over channels, it becomes easier to work with
# the tensor when it is in NCHW format vs NHWC
q = tf.transpose(q, perm=[0, 3, 1, 2])
# Select the conv-net channels at the state position (S1,S2).
# This intuitively corresponds to each channel representing an action, and the convnet the Q function.
# The tricky thing is we want to select the same (S1,S2) position *for each* channel and for each sample
# TODO: performance can be improved here by substituting expensive
# transpose calls with better indexing for gather_nd
bs = tf.shape(q)[0]
rprn = tf.reshape(
tf.tile(tf.reshape(tf.range(bs), [-1, 1]), [1, state_batch_size]),
[-1])
ins1 = tf.cast(tf.reshape(S1, [-1]), tf.int32)
ins2 = tf.cast(tf.reshape(S2, [-1]), tf.int32)
idx_in = tf.transpose(tf.stack([ins1, ins2, rprn]), [1, 0])
q_out = tf.gather_nd(tf.transpose(q, [2, 3, 0, 1]), idx_in, name="q_out")
# softmax output weights
output = tf.nn.softmax(tf.matmul(q_out, w_o), name="output")
return output
def refresh_shortlist():
"""Update the shortlist with the highest scores in id_to_score."""
new_scores, new_ids = tf.nn.top_k(self.id_to_score, self.shortlist_size)
smallest_new_score = tf.reduce_min(new_scores)
new_length = tf.reduce_sum(tf.to_int32(tf.greater(new_scores, tf.float32.min)))
u1 = self.sl_ids.assign(tf.to_int64(tf.concat_v2([[new_length], new_ids], 0)))
u2 = self.sl_scores.assign(tf.concat_v2([[smallest_new_score], new_scores], 0))
self.last_ops = [u1, u2]
return tf.group(u1, u2)
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
correctly_recognised_indices = tf.squeeze(
tf.where(correct_prediction),
[1]) # indices of correctly recognised images
incorrectly_recognised_indices = tf.squeeze(
tf.where(tf.logical_not(correct_prediction)),
[1]) # indices of incorrectly recognised images
everything_incorrect_first = tf.concat_v2(
[incorrectly_recognised_indices, correctly_recognised_indices],
0) # images reordered with indeces of unrecognised images first
everything_incorrect_first = tf.slice(
everything_incorrect_first, [0],
[n]) # compute first 100 only - no space to display more anyway
# compute n=100 digits to display only
Xs = tf.gather(X, everything_incorrect_first)
Ys = tf.gather(Y, everything_incorrect_first)
Ys_ = tf.gather(Y_, everything_incorrect_first)
correct_prediction_s = tf.gather(correct_prediction,
everything_incorrect_first)
digits_left = tf.image.grayscale_to_rgb(
tensorflowvisu_digits.digits_left())
correct_tags = tf.gather(digits_left, tf.argmax(
Ys_, 1)) # correct digits to be printed on the images
digits_right = tf.image.grayscale_to_rgb(
tensorflowvisu_digits.digits_right())
computed_tags = tf.gather(digits_right, tf.argmax(
Ys, 1)) # computed digits to be printed on the images
#superimposed_digits = correct_tags+computed_tags
superimposed_digits = tf.where(
correct_prediction_s, tf.zeros_like(correct_tags),
correct_tags + computed_tags
) # only pring the correct and computed digits on unrecognised images
correct_bkg = tf.reshape(tf.tile([1.3, 1.3, 1.3], [28 * 28]),
[1, 28, 28, 3]) # white background
incorrect_bkg = tf.reshape(tf.tile([1.3, 1.0, 1.0], [28 * 28]),
[1, 28, 28, 3]) # red background
recognised_bkg = tf.gather(
tf.concat_v2([incorrect_bkg, correct_bkg], 0),
tf.cast(correct_prediction_s, tf.int32)
) # pick either the red or the white background depending on recognised status
I = tf.image.grayscale_to_rgb(Xs)
I = (
(1 - (I + superimposed_digits)) * recognised_bkg
) / 1.3 # stencil extra data on top of images and reorder them unrecognised first
I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
Islices = [] # 100 images => 10x10 image block
for imslice in range(lines):
Islices.append(
tf.concat_v2(
tf.unstack(
tf.slice(I, [imslice * n // lines, 0, 0, 0],
[n // lines, 28, 28, 3])), 1))
I = tf.concat_v2(Islices, 0)
return I
def build_model(self):
sc = predictron_arg_scope()
with tf.variable_scope('state'):
with slim.arg_scope(sc):
state = slim.conv2d(self.inputs, 32, [3, 3], scope='conv1')
state = layers.batch_norm(state, activation_fn=tf.nn.relu, scope='conv1/preact')
state = slim.conv2d(state, 32, [3, 3], scope='conv2')
state = layers.batch_norm(state, activation_fn=tf.nn.relu, scope='conv2/preact')
iter_template = tf.make_template('iter', self.iter_func, unique_name_='iter')
rewards_arr = []
gammas_arr = []
lambdas_arr = []
values_arr = []
for k in range(self.max_depth):
state, reward, gamma, lambda_, value = iter_template(state)
rewards_arr.append(reward)
gammas_arr.append(gamma)
lambdas_arr.append(lambda_)
values_arr.append(value)
_, _, _, _, value = iter_template(state)
# K + 1 elements
values_arr.append(value)
bs = tf.shape(self.inputs)[0]
# [batch_size, K * maze_size]
self.rewards = tf.pack(rewards_arr, axis=1)
# [batch_size, K, maze_size]
self.rewards = tf.reshape(self.rewards, [bs, self.max_depth, self.maze_size])
# [batch_size, K + 1, maze_size]
self.rewards = tf.concat_v2(values=[tf.zeros(shape=[bs, 1, self.maze_size], dtype=tf.float32), self.rewards],
axis=1, name='rewards')
# [batch_size, K * maze_size]
self.gammas = tf.pack(gammas_arr, axis=1)
# [batch_size, K, maze_size]
self.gammas = tf.reshape(self.gammas, [bs, self.max_depth, self.maze_size])
# [batch_size, K + 1, maze_size]
self.gammas = tf.concat_v2(values=[tf.ones(shape=[bs, 1, self.maze_size], dtype=tf.float32), self.gammas],
axis=1, name='gammas')
# [batch_size, K * maze_size]
self.lambdas = tf.pack(lambdas_arr, axis=1)
# [batch_size, K, maze_size]
self.lambdas = tf.reshape(self.lambdas, [-1, self.max_depth, self.maze_size])
# [batch_size, (K + 1) * maze_size]
self.values = tf.pack(values_arr, axis=1)
# [batch_size, K + 1, maze_size]
self.values = tf.reshape(self.values, [-1, (self.max_depth + 1), self.maze_size])
self.build_preturns()
self.build_lambda_preturns()
def _phase_shift(I, r):
bsize, a, b, c = I.get_shape().as_list()
bsize = tf.shape(I)[0] # Handling Dimension(None) type for undefined batch dim
X = tf.reshape(I, (bsize, a, b, r, r))
X = tf.transpose(X, (0, 1, 2, 4, 3)) # bsize, a, b, 1, 1
X = tf.split(X, a, 1) # a, [bsize, b, r, r]
X = tf.concat_v2([tf.squeeze(x, axis=1) for x in X], 2) # bsize, b, a*r, r
X = tf.split(X, b, 1) # b, [bsize, a*r, r]
X = tf.concat_v2([tf.squeeze(x, axis=1) for x in X], 2) # bsize, a*r, b*r
return tf.reshape(X, (bsize, a*r, b*r, 1))
def testConcat(self):
tf_val = tf.concat_v2([[16, 37], tf.placeholder(tf.int32, shape=(2,))], 0)
c_val = tensor_util.constant_value_as_shape(tf_val)
self.assertEqual([16, 37, None, None], c_val.as_list())
tf_val = tf.concat_v2(
[[16, 37], tf.placeholder(
tf.int32, shape=(1,)), [48]], 0)
c_val = tensor_util.constant_value_as_shape(tf_val)
self.assertEqual([16, 37, None, 48], c_val.as_list())
def testConcat(self):
tf_val = tf.concat_v2(
[[16, 37], tf.placeholder(tf.int32, shape=(2, ))], 0)
c_val = tensor_util.constant_value_as_shape(tf_val)
self.assertEqual([16, 37, None, None], c_val.as_list())
tf_val = tf.concat_v2(
[[16, 37], tf.placeholder(tf.int32, shape=(1, )), [48]], 0)
c_val = tensor_util.constant_value_as_shape(tf_val)
self.assertEqual([16, 37, None, 48], c_val.as_list())
def forward(self, x, z): #, h_tm1, pz_tm1):
xz = tf.concat_v2([x, tf.expand_dims(z, -1)], 2)
h0 = tf.zeros((1, tf.shape(x)[1], self.h_size))
h = self.rcnn.forward(xz)
h_prev = tf.concat_v2([h0, h[:-1]], 0)
pz = tf.nn.sigmoid(
tf.matmul(tf.reshape(x, [-1, self.in_size]), self.w1) +
tf.matmul(tf.reshape(h_prev, [-1, self.h_size]), self.w2) +
self.bias)
return tf.reshape(pz, [tf.shape(x)[0], tf.shape(x)[1]])
def unpool(x, size):
out = tf.concat_v2([x, tf.zeros_like(x)], 3)
out = tf.concat_v2([out, tf.zeros_like(out)], 2)
sh = x.get_shape().as_list()
if None not in sh[1:]:
out_size = [-1, sh[1] * size, sh[2] * size, sh[3]]
return tf.reshape(out, out_size)
shv = tf.shape(x)
ret = tf.reshape(out, tf.stack([-1, shv[1] * size, shv[2] * size, sh[3]]))
ret.set_shape([None, None, None, sh[3]])
return ret
def refresh_shortlist():
"""Update the shortlist with the highest scores in id_to_score."""
new_scores, new_ids = tf.nn.top_k(self.id_to_score,
self.shortlist_size)
smallest_new_score = tf.reduce_min(new_scores)
new_length = tf.reduce_sum(
tf.to_int32(tf.greater(new_scores, tf.float32.min)))
u1 = self.sl_ids.assign(
tf.to_int64(tf.concat_v2([[new_length], new_ids], 0)))
u2 = self.sl_scores.assign(
tf.concat_v2([[smallest_new_score], new_scores], 0))
self.last_ops = [u1, u2]
return tf.group(u1, u2)
def logp(self, F, Y):
Y = tf.cast(Y, tf.int32)
scaled_bins_left = tf.concat_v2(
[self.bin_edges / self.sigma,
np.array([np.inf])], 0)
scaled_bins_right = tf.concat_v2(
[np.array([-np.inf]), self.bin_edges / self.sigma], 0)
selected_bins_left = tf.gather(scaled_bins_left, Y)
selected_bins_right = tf.gather(scaled_bins_right, Y)
return tf.log(
probit(selected_bins_left - F / self.sigma) -
probit(selected_bins_right - F / self.sigma) + 1e-6)
def update_tensor(V, dim2, val): # Update tensor V, with index(:,dim2[:]) by val[:]
print 'Shapes Recieved in Update: V, dim, val are ==> ',V.get_shape().as_list(), dim2.get_shape().as_list(), val.get_shape().as_list()
val = tf.cast(val, V.dtype)
def body(_, (v, d2, chg)):
print 'Shapes Recieved in Body of Update: v, d2, chg are ==> ', v.get_shape().as_list(), d2.get_shape().as_list(), chg.get_shape().as_list()
d2_int = tf.cast(d2, tf.int32)
if len(chg.get_shape().as_list()) == 0:
chg = [chg]
else:
chg = tf.reshape(chg, shape=[1]+chg.get_shape().as_list())
oob = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg], axis=0), tf.range(0,len(v.get_shape().as_list())), v.get_shape().as_list())
inb = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg, v[d2_int + 1:]], axis=0), tf.constant(0,shape=[len(v.get_shape().as_list())]), v.get_shape().as_list())
return tf.cond(tf.less(d2_int + 1, v.get_shape().as_list()[0]), inb, oob)
def __call__(self, inputs, state, scope='attention_cell'):
"""Long short-term memory cell with attention (LSTMA)."""
with tf.variable_scope(scope, reuse=self.reuse):
state, attns, attn_states = state
attn_states = tf.reshape(attn_states,
[-1, self._attn_length, self._attn_size])
input_size = self._input_size
if input_size is None:
input_size = inputs.get_shape().as_list()[1]
inputs = _linear([inputs, attns],
input_size,
self.reuse,
trainable=self.trainable,
name=scope)
lstm_output, new_state = self._cell(inputs, state)
new_state_cat = tf.concat_v2(helper.flatten_sq(new_state), 1)
new_attns, new_attn_states = _attention(new_state_cat,
attn_states,
True,
self.reuse,
self._attn_size,
self._attn_vec_size,
self._attn_length,
trainable=self.trainable)
with tf.variable_scope("attn_output_projection"):
output = _linear([lstm_output, new_attns],
self._attn_size,
self.reuse,
trainable=self.trainable,
name=scope)
new_attn_states = tf.concat_v2(
[new_attn_states, tf.expand_dims(output, 1)], 1)
new_attn_states = tf.reshape(
new_attn_states, [-1, self._attn_length * self._attn_size])
new_state = (new_state, new_attns, new_attn_states)
if self.layer_norm is not None:
output = self.layer_norm(output,
self.reuse,
trainable=self.trainable,
**self.layer_norm_args)
new_state = self.layer_norm(new_state,
self.reuse,
trainable=self.trainable,
**self.layer_norm_args)
output = _collect_named_outputs(self.outputs_collections,
scope + '_output', output)
new_state = _collect_named_outputs(self.outputs_collections,
scope + '_state', new_state)
return output, new_state
def UnPooling2x2ZeroFilled(x):
# https://github.com/tensorflow/tensorflow/issues/2169
out = tf.concat_v2([x, tf.zeros_like(x)], 3)
out = tf.concat_v2([out, tf.zeros_like(out)], 2)
sh = x.get_shape().as_list()
if None not in sh[1:]:
out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]]
return tf.reshape(out, out_size)
else:
shv = tf.shape(x)
ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]]))
ret.set_shape([None, None, None, sh[3]])
return ret
def update_tensor(V, dim2, val): # Update tensor V, with index(:,dim2[:]) by val[:]
print 'Shapes Recieved in Update: V, dim, val are ==> ',V.get_shape().as_list(), dim2.get_shape().as_list(), val.get_shape().as_list()
val = tf.cast(val, V.dtype)
def body(_, (v, d2, chg)):
print 'Shapes Recieved in Body of Update: v, d2, chg are ==> ', v.get_shape().as_list(), d2.get_shape().as_list(), chg.get_shape().as_list()
d2_int = tf.cast(d2, tf.int32)
if len(chg.get_shape().as_list()) == 0:
chg = [chg]
else:
chg = tf.reshape(chg, shape=[1]+chg.get_shape().as_list())
oob = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg], axis=0), tf.range(0,len(v.get_shape().as_list())), v.get_shape().as_list())
inb = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg, v[d2_int + 1:]], axis=0), tf.constant(0,shape=[len(v.get_shape().as_list())]), v.get_shape().as_list())
return tf.cond(tf.less(d2_int + 1, v.get_shape().as_list()[0]), inb, oob)
def _make_phi(self, F):
"""
A helper function for making predictions. Constructs a probability
matrix where each row output the probability of the corresponding
label, and the rows match the entries of F.
Note that a matrix of F values is flattened.
"""
scaled_bins_left = tf.concat_v2(
[self.bin_edges / self.sigma,
np.array([np.inf])], 0)
scaled_bins_right = tf.concat_v2(
[np.array([-np.inf]), self.bin_edges / self.sigma], 0)
return probit(scaled_bins_left - tf.reshape(F, (-1, 1)) / self.sigma)\
- probit(scaled_bins_right - tf.reshape(F, (-1, 1)) / self.sigma)
def testRandomInitUnevenPartitions(self):
with self.test_session():
rnd = tf.Variable(tf.random_uniform([20, 43], dtype=tf.float64))
var_lists = [
tf.create_partitioned_variables(rnd.get_shape(), [1, i], rnd.initialized_value()) for i in xrange(1, 10)
]
tf.global_variables_initializer().run()
rnd_val = rnd.eval()
# Only check the slice save specs for the first 5 tf.
save_specs = [
# One slice
["20 43 0,20:0,43"],
# Two slices
["20 43 0,20:0,22", "20 43 0,20:22,21"],
# Three slices
["20 43 0,20:0,15", "20 43 0,20:15,14", "20 43 0,20:29,14"],
# Four slices
["20 43 0,20:0,11", "20 43 0,20:11,11", "20 43 0,20:22,11", "20 43 0,20:33,10"],
# Five slices
["20 43 0,20:0,9", "20 43 0,20:9,9", "20 43 0,20:18,9", "20 43 0,20:27,8", "20 43 0,20:35,8"],
]
for i, vs in enumerate(var_lists):
var_val = tf.concat_v2(vs, 1).eval()
self.assertAllClose(rnd_val, var_val)
self.assertEqual([tf.float64] * len(vs), [v.dtype.base_dtype for v in vs])
if i < len(save_specs):
self._TestSaveSpec(vs, save_specs[i])
def unzip(x, split_dim, current_length, num_splits=2, name=None):
"""Splits a tensor by unzipping along the split_dim.
For example the following array split into 2 would be:
[1, 2, 3, 4, 5, 6] -> [1, 3, 5], [2, 4, 6]
and by 3:
[1, 2, 3, 4] -> [1, 4], [2], [3]
Args:
x: The tensor to split.
split_dim: The dimension to split along.
current_length: Current length along the split_dim.
num_splits: The number of splits.
name: Optional name for this op.
Returns:
A length num_splits sequence.
"""
with tf.name_scope(name, "unzip", [x]) as scope:
x = tf.convert_to_tensor(x, name="x")
# There is probably a more efficient way to do this.
all_splits = tf.split(value=x, num_or_size_splits=current_length, axis=split_dim, name=scope)
splits = [[] for _ in xrange(num_splits)]
for i in xrange(current_length):
splits[i % num_splits].append(all_splits[i])
return [tf.concat_v2(s, split_dim) for s in splits]
def create_discriminator(discrim_inputs, discrim_targets):
n_layers = 3
layers = []
# 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2]
input = tf.concat_v2([discrim_inputs, discrim_targets], axis=3)
# layer_1: [batch, 256, 256, in_channels * 2] => [batch, 128, 128, ndf]
with tf.variable_scope("layer_1"):
convolved = conv(input, a.ndf, stride=2)
rectified = lrelu(convolved, 0.2)
layers.append(rectified)
# layer_2: [batch, 128, 128, ndf] => [batch, 64, 64, ndf * 2]
# layer_3: [batch, 64, 64, ndf * 2] => [batch, 32, 32, ndf * 4]
# layer_4: [batch, 32, 32, ndf * 4] => [batch, 31, 31, ndf * 8]
for i in range(n_layers):
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
out_channels = a.ndf * min(2**(i + 1), 8)
stride = 1 if i == n_layers - 1 else 2 # last layer here has stride 1
convolved = conv(layers[-1], out_channels, stride=stride)
normalized = batchnorm(convolved)
rectified = lrelu(normalized, 0.2)
layers.append(rectified)
# layer_5: [batch, 31, 31, ndf * 8] => [batch, 30, 30, 1]
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
convolved = conv(rectified, out_channels=1, stride=1)
output = tf.sigmoid(convolved)
layers.append(output)
return layers[-1]
def merge_outputs(tensor_list, name="MergeOutputs"):
""" Merge Outputs.
A layer that concatenate all outputs of a network into a single tensor.
Input:
List of Tensors [_shape_].
Output:
Concatenated Tensors [nb_tensors, _shape_].
Arguments:
tensor_list: list of `Tensor`. The network outputs.
name: `str`. A name for this layer (optional).
Returns:
A `Tensor`.
"""
with tf.name_scope(name) as scope:
x = tf.concat_v2(tensor_list, 1)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, x)
return x
def conv_cond_concat(x, y):
"""Concatenate conditioning vector on feature map axis."""
x_shapes = x.get_shape()
y_shapes = y.get_shape()
return tf.concat_v2(
[x, y * tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])],
3)
def testConcat(self):
np_val = np.random.rand(3, 4, 7).astype(np.float32)
tf_val = tf.concat_v2(
[np_val[0:1, :, :], np_val[1:2, :, :], np_val[2:3, :, :]], 0)
c_val = tf.contrib.util.constant_value(tf_val)
self.assertAllClose(np_val, c_val)
tf_val = tf.concat_v2([np_val[0, :, :], np_val[1, :, :], np_val[2, :, :]],
tf.placeholder(tf.int32))
c_val = tf.contrib.util.constant_value(tf_val)
self.assertIs(None, c_val)
tf_val = tf.concat_v2(
[np_val[0, :, :], tf.placeholder(tf.float32), np_val[2, :, :]], 1)
c_val = tf.contrib.util.constant_value(tf_val)
self.assertIs(None, c_val)
def concat(input_layer, concat_dim, other_tensors=None):
"""Concatenates input PrettyTensor with other_tensors along the specified dim.
This adds the Pretty Tensor passed via input_layer to the front of the list of
tensors to concat.
Args:
input_layer: The input layer.
concat_dim: The dimension along which to concat.
other_tensors: The tensors to concatenate with as an iterable or None if
this is called on a sequence.
Returns:
A new PrettyTensor.
Raises:
ValueError: If other_tensors is None and this is not a sequence.
"""
if input_layer.is_sequence():
all_tensors = input_layer.sequence
all_tensors.extend(other_tensors or [])
else:
all_tensors = [input_layer]
if other_tensors is None:
raise ValueError("Other Tensors must be supplied.")
all_tensors.extend(other_tensors)
# Edge cases really only apply when this is a sequence with 0 or 1 element.
if not all_tensors:
return prettytensor.wrap_sequence([])
else:
return tf.concat_v2(all_tensors, concat_dim)
def one_hot_mask(labels, num_classes, scope=None):
"""Compute 1-hot encodings for masks.
Given a label image, this computes the one hot encoding at
each pixel.
Args:
labels: (batch_size, width, height, 1) tensor containing labels.
num_classes: number of classes
scope: optional scope name
Returns:
Tensor of shape (batch_size, width, height, num_classes) with
a 1-hot encoding.
"""
with tf.name_scope(scope, "OneHotMask", [labels]):
height, width, depth = _shape(labels)
assert depth == 1
sparse_labels = tf.to_int32(tf.reshape(labels, [-1, 1]))
sparse_size, _ = _shape(sparse_labels)
indices = tf.reshape(tf.range(0, sparse_size, 1), [-1, 1])
concated = tf.concat_v2([indices, sparse_labels], 1)
dense_result = tf.sparse_to_dense(concated, [sparse_size, num_classes], 1.0,
0.0)
result = tf.reshape(dense_result, [height, width, num_classes])
return result
def horizontal_lstm(images, num_filters_out, scope=None):
"""Run an LSTM bidirectionally over all the rows of each image.
Args:
images: (num_images, height, width, depth) tensor
num_filters_out: output depth
scope: optional scope name
Returns:
(num_images, height, width, num_filters_out) tensor, where
num_steps is width and new num_batches is num_image_batches * height
"""
with tf.variable_scope(scope, "HorizontalLstm", [images]):
batch_size, _, _, _ = _shape(images)
sequence = images_to_sequence(images)
with tf.variable_scope("lr"):
hidden_sequence_lr = lstm1d.ndlstm_base(sequence, num_filters_out // 2)
with tf.variable_scope("rl"):
hidden_sequence_rl = (
lstm1d.ndlstm_base(sequence,
num_filters_out - num_filters_out // 2,
reverse=1))
output_sequence = tf.concat_v2([hidden_sequence_lr, hidden_sequence_rl], 2)
output = sequence_to_images(output_sequence, batch_size)
return output
def _average_gradients(tower_grads):
"""Calculate the average gradient for each shared variable across all towers.
Note that this function provides a synchronization point across all towers.
Args:
tower_grads: List of lists of (gradient, variable) tuples. The outer list
is over individual gradients. The inner list is over the gradient
calculation for each tower.
Returns:
List of pairs of (gradient, variable) where the gradient has been averaged
across all towers.
"""
average_grads = []
for grad_and_vars in zip(*tower_grads):
# Note that each grad_and_vars looks like the following:
# ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
grads = []
for g, _ in grad_and_vars:
# Add 0 dimension to the gradients to represent the tower.
expanded_g = tf.expand_dims(g, 0)
# Append on a 'tower' dimension which we will average over below.
grads.append(expanded_g)
# Average over the 'tower' dimension.
grad = tf.concat_v2(grads, 0)
grad = tf.reduce_mean(grad, 0)
# Keep in mind that the Variables are redundant because they are shared
# across towers. So .. we will just return the first tower's pointer to
# the Variable.
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
"""Builds the 35x35 resnet block."""
with tf.variable_scope(scope, 'Block35', [net], reuse=reuse):
with tf.variable_scope('Branch_0'):
tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1')
with tf.variable_scope('Branch_1'):
tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
tower_conv1_1 = slim.conv2d(tower_conv1_0,
32,
3,
scope='Conv2d_0b_3x3')
with tf.variable_scope('Branch_2'):
tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
tower_conv2_1 = slim.conv2d(tower_conv2_0,
48,
3,
scope='Conv2d_0b_3x3')
tower_conv2_2 = slim.conv2d(tower_conv2_1,
64,
3,
scope='Conv2d_0c_3x3')
mixed = tf.concat_v2([tower_conv, tower_conv1_1, tower_conv2_2], 3)
up = slim.conv2d(mixed,
net.get_shape()[3],
1,
normalizer_fn=None,
activation_fn=None,
scope='Conv2d_1x1')
net += scale * up
if activation_fn:
net = activation_fn(net)
return net
def test_broadcast_apply_and_solve(self):
# These cannot be done in the automated (base test class) tests since they
# test shapes that tf.matmul cannot handle.
# In particular, tf.matmul does not broadcast.
with self.test_session() as sess:
x = tf.random_normal(shape=(2, 2, 3, 4))
# This LinearOperatorDiag will be brodacast to (2, 2, 3, 3) during solve
# and apply with 'x' as the argument.
diag = tf.random_uniform(shape=(2, 1, 3))
operator = linalg.LinearOperatorDiag(diag, is_self_adjoint=True)
self.assertAllEqual((2, 1, 3, 3), operator.shape)
# Create a batch matrix with the broadcast shape of operator.
diag_broadcast = tf.concat_v2((diag, diag), 1)
mat = tf.matrix_diag(diag_broadcast)
self.assertAllEqual((2, 2, 3, 3), mat.get_shape()) # being pedantic.
operator_apply = operator.apply(x)
mat_apply = tf.matmul(mat, x)
self.assertAllEqual(operator_apply.get_shape(), mat_apply.get_shape())
self.assertAllClose(*sess.run([operator_apply, mat_apply]))
operator_solve = operator.solve(x)
mat_solve = tf.matrix_solve(mat, x)
self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape())
self.assertAllClose(*sess.run([operator_solve, mat_solve]))
def average_gradients(tower_grads):
"""Calculate the average gradient for each shared variable across all towers.
Note that this function provides a synchronization point across all towers.
Args:
tower_grads: List of lists of (gradient, variable) tuples. The outer list
is over individual gradients. The inner list is over the gradient
calculation for each tower.
Returns:
List of pairs of (gradient, variable) where the gradient has been averaged
across all towers.
"""
average_grads = []
for grad_and_vars in zip(*tower_grads):
# Note that each grad_and_vars looks like the following:
# ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
grads = []
for g, _ in grad_and_vars:
# Add 0 dimension to the gradients to represent the tower.
expanded_g = tf.expand_dims(g, 0)
# Append on a 'tower' dimension which we will average over below.
grads.append(expanded_g)
# Average over the 'tower' dimension.
grad = tf.concat_v2(grads, 0)
grad = tf.reduce_mean(grad, 0)
# Keep in mind that the Variables are redundant because they are shared
# across towers. So .. we will just return the first tower's pointer to
# the Variable.
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
def PS(X, r, color=False):
if color:
Xc = tf.split(X, 3, 3)
X = tf.concat_v2([_phase_shift(x, r) for x in Xc], 3)
else:
X = _phase_shift(X, r)
return X
def testDynamicAttentionDecoderStateIsTuple(self):
with self.test_session() as sess:
with tf.variable_scope("root",
initializer=tf.constant_initializer(0.5)):
cell = tf.contrib.rnn.BasicLSTMCell(2, state_is_tuple=True)
cell = tf.contrib.rnn.MultiRNNCell(cells=[cell] * 2,
state_is_tuple=True)
inp = tf.constant(0.5, shape=[2, 2, 2])
enc_outputs, enc_state = tf.contrib.rnn.static_rnn(
cell, inp, dtype=tf.float32)
attn_states = tf.concat_v2(
[tf.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs],
1)
dec_inp = [tf.constant(0.4, shape=[2, 2])] * 3
dec, mem = tf.contrib.legacy_seq2seq.attention_decoder(
dec_inp, enc_state,
attn_states, cell, output_size=4)
sess.run([tf.global_variables_initializer()])
res = sess.run(dec)
self.assertEqual(3, len(res))
self.assertEqual((2, 4), res[0].shape)
res = sess.run([mem])
self.assertEqual(2, len(res[0]))
self.assertEqual((2, 2), res[0][0].c.shape)
self.assertEqual((2, 2), res[0][0].h.shape)
self.assertEqual((2, 2), res[0][1].c.shape)
self.assertEqual((2, 2), res[0][1].h.shape)
def linear(args, input_size, output_size, bias, bias_start=0.0, scope=None):
"""Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Args:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
bias: boolean, whether to add a bias term or not.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
"""
if not isinstance(args, (list, tuple)):
args = [args]
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = []
for a in args:
try:
shapes.append(a.get_shape().as_list())
except Exception as e:
shapes.append(a.shape)
is_vector = False
for idx, shape in enumerate(shapes):
if len(shape) != 2:
is_vector = True
args[idx] = tf.reshape(args[idx], [1, -1])
total_arg_size = input_size
"""for idx, shape in enumerate(shapes):
if len(shape) != 2:
is_vector = True
args[idx] = tf.reshape(args[idx], [1, -1])
total_arg_size += shape[0]
else:
total_arg_size += shape[1]"""
# Now the computation.
with vs.variable_scope(scope or "Linear"):
#initializer = tf.random_normal_initializer(mean=0., stddev=1.,)
#matrix = vs.get_variable("Matrix", [total_arg_size, output_size], initializer=initializer)
matrix = vs.get_variable("Matrix", [total_arg_size, output_size])
if len(args) == 1:
res = tf.matmul(args[0], matrix)
else:
res = tf.matmul(tf.concat_v2(args, 1), matrix)
if not bias:
return res
bias_term = vs.get_variable(
"Bias", [output_size],
initializer=init_ops.constant_initializer(bias_start))
if is_vector:
return tf.reshape(res + bias_term, [-1])
else:
return res + bias_term
def testSegmentMaxGradientWithTies(self):
inputs = tf.constant([1.0], dtype=tf.float32)
data = tf.concat_v2([inputs, inputs], 0)
segment_ids = tf.constant([0, 0], dtype=tf.int64)
segment_max = tf.segment_max(data, segment_ids)
with self.test_session():
error = tf.test.compute_gradient_error(inputs, [1], segment_max, [1])
self.assertLess(error, 1e-4)
def _inference(self, docs, queries):
"""
Computes document attentions given a document batch and query batch.
"""
with tf.name_scope("inference"):
# Compute document lengths / query lengths for batch
doc_lens = length(docs)
query_lens = length(queries)
batch_size = tf.shape(docs)[0]
with tf.variable_scope('encode'):
# Encode Document / Query
with tf.variable_scope('docs'), tf.device('/gpu:0'):
encoded_docs = tf.nn.dropout(self._embed(docs), self._keep_prob)
encoded_docs = self._bidirectional_encode(encoded_docs, doc_lens, self._encode_size)
with tf.variable_scope('queries'), tf.device('/gpu:1'):
encoded_queries = tf.nn.dropout(self._embed(queries), self._keep_prob)
encoded_queries = self._bidirectional_encode(encoded_queries, query_lens, self._encode_size)
with tf.variable_scope('attend') as scope:
infer_gru = tf.nn.rnn_cell.GRUCell(self._infer_size)
infer_state = infer_gru.zero_state(batch_size, tf.float32)
for iter_step in range(self._num_glimpses):
if iter_step > 0:
scope.reuse_variables()
# Glimpse query and document
with tf.device('/gpu:0'):
q_attention, q_glimpse = self._glimpse(self._A_q, self._a_q, encoded_queries, infer_state)
tf.add_to_collection('query_attentions', q_attention)
with tf.device('/gpu:1'):
d_attention, d_glimpse = self._glimpse(self._A_d, self._a_d, encoded_docs, tf.concat_v2([infer_state, q_glimpse], 1))
tf.add_to_collection('doc_attentions', d_attention)
# Search Gates
gate_concat = tf.concat_v2([infer_state, q_glimpse, d_glimpse, q_glimpse * d_glimpse], 1)
r_d = tf.sigmoid(tf.matmul(gate_concat, self._g_d))
r_d = tf.nn.dropout(r_d, self._keep_prob)
r_q = tf.sigmoid(tf.matmul(gate_concat, self._g_q))
r_q = tf.nn.dropout(r_q, self._keep_prob)
combined_gated_glimpse = tf.concat_v2([r_q * q_glimpse, r_d * d_glimpse], 1)
_, infer_state = infer_gru(combined_gated_glimpse, infer_state)
return tf.to_float(tf.sign(tf.abs(docs))) * d_attention
def LSTMCell(cls, x, mprev, cprev, weights):
xm = tf.concat_v2([x, mprev], 1)
i_i, i_g, f_g, o_g = tf.split(
value=tf.matmul(xm, weights), num_or_size_splits=4, axis=1)
new_c = tf.sigmoid(f_g) * cprev + tf.sigmoid(i_g) * tf.tanh(i_i)
new_c = tf.clip_by_value(new_c, -50.0, 50.0)
new_m = tf.sigmoid(o_g) * tf.tanh(new_c)
return new_m, new_c
def __call__(self, input_tuple, state, scope=None):
with vs.variable_scope(scope or type(self).__name__):
self._init_parameters()
_input, target = input_tuple
assert _input.get_shape()[2] == 1, "Need input depth == 1"
_input = tf.squeeze(_input, 2)
u, s, r, inner_spikes, spike_history, dW, dF, dR = state
_input_filtered = tf.matmul(_input, self.F)
s = (1.0 - tau_syn) * s + tf.concat_v2([_input_filtered, inner_spikes], 1)
u = (1.0 - tau_mem) * u + mo.matmul(s, self.W)
r = (1.0 - tau_refr) * r
act_raw = self._activation(u)
act = act_raw * tf.exp(-r)
spikes = tf.where(
act > tf.random_uniform([batch_size, self._num_units]),
tf.ones([batch_size, self._num_units]),
tf.zeros([batch_size, self._num_units])
)
r += spikes * amp_refr
spike_history = cat_hist(spike_history, spikes, 1)
act_grad = tf.gradients([act], [u])[0]
neuron_derivative = (act_grad/act_raw) * (spikes - act)
spike_history_reshaped = tf.reshape(tf.transpose(spike_history, [0, 2, 1]), [batch_size * self._num_units, L])
output_point = tf.matmul(spike_history_reshaped, self.R)
output_point = tf.reshape(output_point, [batch_size, self._num_units])
output_point = tf.reduce_sum(output_point, 1)
output_point = tf.expand_dims(output_point, 1)
loss = tf.reduce_sum(tf.square(output_point - target), 1)/2.0
loss = tf.expand_dims(loss, 1)
dR += - lrate * tf.gradients([loss], [self.R])[0]
sp = - tf.gradients([loss], [spike_history])[0]
sp = tf.reduce_sum(sp, [1,2])
sp = tf.expand_dims(sp, 1)
dW += lrate * outer(
tf.reduce_sum(s, 0),
tf.reduce_sum(neuron_derivative * sp, 0)
)
return GLMOutputTuple(spikes, act, output_point, loss), GLMStateTuple(u, s, r, spikes, spike_history, dW, dF, dR)
def testDegenerate(self):
with self.test_session():
rnd = tf.Variable(tf.random_uniform([10, 43]))
vs = tf.create_partitioned_variables(rnd.get_shape(), [1, 1], rnd.initialized_value())
tf.global_variables_initializer().run()
val = tf.concat_v2(vs, 0).eval()
rnd = rnd.eval()
self.assertAllClose(rnd, val)
self._TestSaveSpec(vs, ["10 43 0,10:0,43"])
def remove(self, ids):
"""Remove the ids (and their associated scores) from the TopN."""
with tf.control_dependencies(self.last_ops):
scatter_op = tf.scatter_update(self.id_to_score, ids, tf.ones_like(ids, dtype=tf.float32) * tf.float32.min)
# We assume that removed ids are almost always in the shortlist,
# so it makes no sense to hide the Op behind a tf.cond
shortlist_ids_to_remove, new_length = tensor_forest_ops.top_n_remove(self.sl_ids, ids)
u1 = tf.scatter_update(
self.sl_ids,
tf.concat_v2([[0], shortlist_ids_to_remove], 0),
tf.concat_v2([new_length, tf.ones_like(shortlist_ids_to_remove) * -1], 0),
)
u2 = tf.scatter_update(
self.sl_scores,
shortlist_ids_to_remove,
tf.float32.min * tf.ones_like(shortlist_ids_to_remove, dtype=tf.float32),
)
self.last_ops = [scatter_op, u1, u2]
def test(self):
condition = core.LabeledTensor(tf.range(5) < 3, ['x'])
x = core.LabeledTensor(tf.ones(5), ['x'])
y = core.LabeledTensor(tf.zeros(5), ['x'])
where_lt = ops.where(condition, x, y)
golden_lt = core.LabeledTensor(
tf.concat_v2([tf.ones(3), tf.zeros(2)], 0), ['x'])
self.assertLabeledTensorsEqual(where_lt, golden_lt)
def __call__(self, x, h, scope=None):
x_t = fun(x, nout = x_transformed, act = tf.nn.tanh, name = "x_transformed", weight_factor = weight_factor, layers_num = layers_num)
# prior, depends only on state
prior_t = fun(h, nout = prior_interm, act = tf.nn.tanh, name = "prior", weight_factor = weight_factor, layers_num = layers_num)
prior_mu_t = fun(prior_t, nout = z_dim, act = tf.identity, name = "prior_mu", weight_factor = weight_factor)
prior_sigma_t = fun(prior_t, nout = z_dim, act = tf.nn.softplus, name = "prior_sigma", weight_factor = weight_factor)
# phi
phi_t = fun(x_t, h, nout = phi_interm, act = tf.nn.tanh, name = "phi", weight_factor = weight_factor, layers_num = layers_num)
phi_mu_t = fun(phi_t, nout = z_dim, act = tf.identity, name = "phi_mu", weight_factor = weight_factor)
phi_sigma_t = fun(phi_t, nout = z_dim, act = tf.nn.softplus, name = "phi_sigma", weight_factor = weight_factor)
# z generating
epsilon = tf.random_normal(tf.shape(z_dim), name='epsilon')
if not self._generator:
z = phi_mu_t + phi_sigma_t * epsilon
else:
z = prior_mu_t + prior_sigma_t * epsilon
z_t = fun(z, nout = z_interm, act = tf.nn.tanh, name = "z_transformed", weight_factor = weight_factor, layers_num = layers_num)
output_t = fun(z_t, h, nout = out_interm, act = tf.nn.tanh, name = "out_transform", weight_factor = weight_factor, layers_num = layers_num)
output_mu = fun(output_t, nout = 1, act =tf.identity, name = "out_mu", weight_factor = weight_factor)
# output_sigma = fun(out_t, nout = 1, act = tf.nn.softplus, name = "out_sigma", weight_factor = weight_factor)
if not self._generator:
x_c = tf.concat_v2([x_t, z_t], 1)
else:
x_t_gen = fun(output_mu, nout = x_transformed, act = tf.nn.tanh, name = "x_transformed", weight_factor = weight_factor, layers_num = layers_num)
x_c = tf.concat_v2([x_t_gen, z_t], 1)
_, new_h = self._base_cell(x_c, h)
return VAEOutputTuple(
prior_mu_t,
prior_sigma_t,
phi_mu_t,
phi_sigma_t,
z,
z_t,
output_mu), new_h
def testVecConstantInit(self):
with self.test_session():
rnd_par = tf.constant([1, 2, 3, 4])
vs = tf.create_partitioned_variables([4], [4], rnd_par)
tf.global_variables_initializer().run()
val = tf.concat_v2(vs, 0).eval()
rnd = rnd_par.eval()
self.assertAllClose(rnd, val)
self.assertEqual([tf.int32] * 4, [v.dtype.base_dtype for v in vs])
self._TestSaveSpec(vs, ["4 0,1", "4 1,1", "4 2,1", "4 3,1"])
def testConstantInit(self):
with self.test_session():
rnd_par = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]])
vs = tf.create_partitioned_variables([2, 4], [1, 2], rnd_par)
tf.global_variables_initializer().run()
val = tf.concat_v2(vs, 1).eval()
rnd = rnd_par.eval()
self.assertAllClose(rnd, val)
self.assertEqual([tf.int32] * 2, [v.dtype.base_dtype for v in vs])
self._TestSaveSpec(vs, ["2 4 0,2:0,2", "2 4 0,2:2,2"])
def testConcatLargeTensors(self):
# CPU-only test, because it fails on GPUs with <= 4GB memory.
with tf.device("/cpu:0"):
a = tf.ones([2**31 + 6], dtype=tf.int8)
b = tf.zeros([1024], dtype=tf.int8)
onezeros = tf.concat_v2([a, b], 0)
with self.test_session(use_gpu=False):
# TODO(dga): Add more depth to this test to validate correctness,
# not just non-crashingness, once other large tensor fixes have gone in.
_ = onezeros.eval()