def dot(a, b):
with tf.op_scope([a, b], 'dot'):
# TODO: implement N-dimensinal dot product that consistent with Numpy.
a_shape = a.get_shape().as_list()
a_dims = len(a_shape)
b_shape = b.get_shape().as_list()
b_dims = len(b_shape)
# scalar dot scalar, scalar dot tensor or tensor dot scalar: just do element-wise multiply.
if a_dims == 0 or b_dims == 0:
return a * b
# vector dot vector, where we can just perform element-wise prod, and then sum them all.
if a_dims == 1 and b_dims == 1:
return tf.reduce_sum(a * b)
# vector dot matrix or matrix dot vector, where we should expand the vector to matrix, and then squeeze result.
if a_dims <= 2 and b_dims <= 2:
if a_dims == 1:
a = tf.expand_dims(a, dim=0)
if b_dims == 1:
b = tf.expand_dims(b, dim=1)
ret = tf.matmul(a, b)
if a_dims == 1:
ret = tf.squeeze(ret, [0])
if b_dims == 1:
ret = tf.squeeze(ret, [1])
return ret
# throw exception, that we do not know how to handle the situation.
raise TypeError('Tensor dot between shape %r and %r is not supported.' % (a_shape, b_shape))
def from_logits(behaviour_policy_logits,
target_policy_logits,
actions,
discounts,
rewards,
values,
bootstrap_value,
clip_rho_threshold=1.0,
clip_pg_rho_threshold=1.0,
name='vtrace_from_logits'):
"""multi_from_logits wrapper used only for tests"""
res = multi_from_logits(
[behaviour_policy_logits], [target_policy_logits], [actions],
discounts,
rewards,
values,
bootstrap_value,
clip_rho_threshold=clip_rho_threshold,
clip_pg_rho_threshold=clip_pg_rho_threshold,
name=name)
return VTraceFromLogitsReturns(
vs=res.vs,
pg_advantages=res.pg_advantages,
log_rhos=res.log_rhos,
behaviour_action_log_probs=tf.squeeze(
res.behaviour_action_log_probs, axis=0),
target_action_log_probs=tf.squeeze(
res.target_action_log_probs, axis=0),
)
def get_reconstructed_image(self, real, imag, name=None):
"""
:param real:
:param imag:
:param name:
:return:
"""
complex_k_space_label = tf.complex(real=tf.squeeze(real), imag=tf.squeeze(imag), name=name+"_complex_k_space")
rec_image_complex = tf.expand_dims(tf.ifft2d(complex_k_space_label), axis=1)
rec_image_real = tf.reshape(tf.real(rec_image_complex), shape=[-1, 1, self.dims_out[1], self.dims_out[2]])
rec_image_imag = tf.reshape(tf.imag(rec_image_complex), shape=[-1, 1, self.dims_out[1], self.dims_out[2]])
# Shifting
top, bottom = tf.split(rec_image_real, num_or_size_splits=2, axis=2)
top_left, top_right = tf.split(top, num_or_size_splits=2, axis=3)
bottom_left, bottom_right = tf.split(bottom, num_or_size_splits=2, axis=3)
top_shift = tf.concat(axis=3, values=[bottom_right, bottom_left])
bottom_shift = tf.concat(axis=3, values=[top_right, top_left])
shifted_image = tf.concat(axis=2, values=[top_shift, bottom_shift])
# Shifting
top_imag, bottom_imag = tf.split(rec_image_imag, num_or_size_splits=2, axis=2)
top_left_imag, top_right_imag = tf.split(top_imag, num_or_size_splits=2, axis=3)
bottom_left_imag, bottom_right_imag = tf.split(bottom_imag, num_or_size_splits=2, axis=3)
top_shift_imag = tf.concat(axis=3, values=[bottom_right_imag, bottom_left_imag])
bottom_shift_imag = tf.concat(axis=3, values=[top_right_imag, top_left_imag])
shifted_image_imag = tf.concat(axis=2, values=[top_shift_imag, bottom_shift_imag])
shifted_image_two_channels = tf.stack([shifted_image[:,0,:,:], shifted_image_imag[:,0,:,:]], axis=1)
return shifted_image_two_channels
def step_loss(self, state, action, time):
# cost:
x_h = tf.slice(state, [0, self.x_h_field[0]], [-1, 1])
x_t = tf.slice(state, [0, self.x_t_field[0]], [-1, self.n_t])
# 0. smooth acceleration policy
cost_accel = tf.square(action)
cost_accel_d = tf.mul(tf.pow(self.gamma, time), cost_accel)
# 1. forcing the host to move forward (until the right point of the roundabout)
cost_prog = tf.square(self.x_goal - x_h)
cost_prog_d = tf.mul(tf.pow(self.gamma, time), cost_prog)
cost_prog_d = tf.squeeze(cost_prog_d, squeeze_dims=[1])
# 2. keeping distance from vehicles ahead
# distance to other vehicles
x_abs_diffs = tf.abs(x_h - x_t)
# punish only vehicles closer than "require distance"
cost_acci = tf.nn.relu(self.require_distance - x_abs_diffs)
# punish only w.r.t vehicles ahead
cost_acci = tf.mul(cost_acci, tf.to_float(x_h < x_t))
# sum over all vehicles
cost_acci = tf.reduce_sum(cost_acci)
# punish only when host is inside the roundabout (or very close to enter)
cost_acci = tf.mul(cost_acci, tf.to_float(x_h > -0.5 * self.host_length))
cost_acci_d = tf.mul(tf.pow(self.gamma, time), cost_acci)
cost_acci_d = tf.squeeze(cost_acci_d, squeeze_dims=[1])
return tf.transpose(tf.pack(values=[cost_accel_d, cost_prog_d, cost_acci_d], name='scan_return'))
def construct_embedding(self):
"""Builds an embedding function on top of images.
Method to be overridden by implementations.
Returns:
embeddings: A 2-d float32 `Tensor` of shape [batch_size, embedding_size]
holding the embedded images.
"""
with tf.variable_scope('tcn_net', reuse=self._reuse) as vs:
self._adaptation_scope = vs.name
net = self._pretrained_output
# Define some adaptation blocks on top of the pre-trained resnet output.
adaptation_blocks = []
adaptation_block_params = [map(
int, i.split('_')) for i in self._config.adaptation_blocks.split('-')]
for i, (depth, num_units) in enumerate(adaptation_block_params):
block = resnet_v2.resnet_v2_block(
'adaptation_block_%d' % i, base_depth=depth, num_units=num_units,
stride=1)
adaptation_blocks.append(block)
# Stack them on top of the resent output.
net = resnet_utils.stack_blocks_dense(
net, adaptation_blocks, output_stride=None)
# Average pool the output.
net = tf.reduce_mean(net, [1, 2], name='adaptation_pool', keep_dims=True)
if self._config.emb_connection == 'fc':
# Use fully connected layer to project to embedding layer.
fc_hidden_sizes = self._config.fc_hidden_sizes
if fc_hidden_sizes == 'None':
fc_hidden_sizes = []
else:
fc_hidden_sizes = map(int, fc_hidden_sizes.split('_'))
fc_hidden_keep_prob = self._config.dropout.keep_fc
net = tf.squeeze(net)
for fc_hidden_size in fc_hidden_sizes:
net = slim.layers.fully_connected(net, fc_hidden_size)
if fc_hidden_keep_prob < 1.0:
net = slim.dropout(net, keep_prob=fc_hidden_keep_prob,
is_training=self._is_training)
# Connect last FC layer to embedding.
embedding = slim.layers.fully_connected(net, self._embedding_size,
activation_fn=None)
else:
# Use 1x1 conv layer to project to embedding layer.
embedding = slim.conv2d(
net, self._embedding_size, [1, 1], activation_fn=None,
normalizer_fn=None, scope='embedding')
embedding = tf.squeeze(embedding)
# Optionally L2 normalize the embedding.
if self._embedding_l2:
embedding = tf.nn.l2_normalize(embedding, dim=1)
return embedding
def __init__(self, memory_cells, query, project_query=False):
"""Define Attention.
Args:
memory_cells (SequenceBatch): a SequenceBatch containing a Tensor of shape (batch_size, num_cells, cell_dim)
query (Tensor): a tensor of shape (batch_size, query_dim).
project_query (bool): defaults to False. If True, the query goes through an extra projection layer to
coerce it to cell_dim.
"""
cell_dim = memory_cells.values.get_shape().as_list()[2]
if project_query:
# project the query up/down to cell_dim
self._projection_layer = Dense(cell_dim, activation='linear')
query = self._projection_layer(query) # (batch_size, cand_dim)
memory_values, memory_mask = memory_cells.values, memory_cells.mask
# batch matrix multiply to compute logit scores for all choices in all batches
query = tf.expand_dims(query, 2) # (batch_size, cell_dim, 1)
logit_values = tf.batch_matmul(memory_values, query) # (batch_size, num_cells, 1)
logit_values = tf.squeeze(logit_values, [2]) # (batch_size, num_cells)
# set all pad logits to negative infinity
logits = SequenceBatch(logit_values, memory_mask)
logits = logits.with_pad_value(-float('inf'))
# normalize to get probs
probs = tf.nn.softmax(logits.values) # (batch_size, num_cells)
retrieved = tf.batch_matmul(tf.expand_dims(probs, 1), memory_values) # (batch_size, 1, cell_dim)
retrieved = tf.squeeze(retrieved, [1]) # (batch_size, cell_dim)
self._logits = logits.values
self._probs = probs
self._retrieved = retrieved
def entropy(self, n, p):
# Note that given n and p where p is a probability vector of
# length k, the entropy requires a sum over all
# possible configurations of a k-vector which sums to n. It's
# expensive.
# http://stackoverflow.com/questions/36435754/generating-a-numpy-array-with-all-combinations-of-numbers-that-sum-to-less-than
sess = tf.Session()
n = sess.run(tf.cast(tf.squeeze(n), dtype=tf.int32))
sess.close()
p = tf.cast(tf.squeeze(p), dtype=tf.float32)
if isinstance(n, np.int32):
k = get_dims(p)[0]
max_range = np.zeros(k, dtype=np.int32) + n
x = np.array([i for i in product(*(range(i+1) for i in max_range))
if sum(i)==n])
logpmf = self.logpmf(x, n, p)
return tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))
else:
out = []
for j in range(n.shape[0]):
k = get_dims(p)[0]
max_range = np.zeros(k, dtype=np.int32) + n[j]
x = np.array([i for i in product(*(range(i+1) for i in max_range))
if sum(i)==n[j]])
logpmf = self.logpmf(x, n[j], p[j, :])
out += [tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))]
return tf.pack(out)
def testExpandAndSqueeze(self):
with self.cached_session():
# TODO(aselle): sparse_split, sparse_reduce_sum,
# sparse_reduce_sum_sparse, reduce_join
a = [[1, 2, 3]]
self.assertAllEqual(tf.expand_dims(tf.squeeze(a, [0]), 0).eval(),
a)
self.assertAllEqual(tf.squeeze(tf.expand_dims(a, 1), [1]).eval(),
a)
self.assertAllEqual(
tf.expand_dims(
tf.squeeze(
[[1, 2, 3]], squeeze_dims=[0]), dim=0).eval(),
a)
self.assertAllEqual(
tf.squeeze(
tf.expand_dims(
[[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
a)
self.assertAllEqual(
tf.squeeze(
tf.expand_dims(
[[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
a)
def _inference(self, x, site, dropout):
# Get each image from the pair
print(x.get_shape())
x_0 = tf.squeeze(x[:, :, :, 0])
x_1 = tf.squeeze(x[:, :, :, 1])
# Share weights between the two models of the pair
with tf.variable_scope("siamese") as scope:
model0 = self.build_model(x_0)
scope.reuse_variables()
model1 = self.build_model(x_1)
# Dot product layer
x = self.corr_layer(model0, model1)
N, M, F = x.get_shape()
x = tf.reshape(x, [int(N), int(M*F)])
site = tf.expand_dims(site, axis=1)
x = tf.concat(1, [x, site])
for i, M in enumerate(self.M[:-1]):
with tf.variable_scope('fc{}'.format(i + 1)):
x = tf.nn.dropout(x, dropout)
x = self.fc(x, M)
# Logits linear layer
with tf.variable_scope('logits'):
x = tf.nn.dropout(x, dropout)
x = self.fc(x, self.M[-1], relu=False)
return tf.squeeze(x) # tf.sigmoid(x)
def logpmf(self, x, n, p):
x = tf.cast(tf.squeeze(x), dtype=tf.float32)
n = tf.cast(tf.squeeze(n), dtype=tf.float32)
p = tf.cast(tf.squeeze(p), dtype=tf.float32)
return log_gamma(n + 1.0) - log_gamma(x + 1.0) - \
log_gamma(n - x + 1.0) + \
tf.mul(x, tf.log(p)) + tf.mul(n - x, tf.log(1.0-p))
def testSlowVsFast(self):
model, features = get_model(transformer.transformer_small())
decode_length = 3
out_logits, _ = model(features)
out_logits = tf.squeeze(out_logits, axis=[2, 3])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
labels=tf.reshape(features["targets"], [-1]))
loss = tf.reduce_mean(loss)
apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)
with self.test_session():
tf.global_variables_initializer().run()
for _ in range(100):
apply_grad.run()
model.set_mode(tf.estimator.ModeKeys.PREDICT)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
greedy_result = model._slow_greedy_infer(
features, decode_length)["outputs"]
greedy_result = tf.squeeze(greedy_result, axis=[2, 3])
fast_result = model._greedy_infer(features, decode_length)["outputs"]
with self.test_session():
greedy_res = greedy_result.eval()
fast_res = fast_result.eval()
self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
self.assertAllClose(greedy_res, fast_res)
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1]) # indices of correctly recognised images
incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images
everything_incorrect_first = tf.concat([incorrectly_recognised_indices, correctly_recognised_indices], 0) # images reordered with indeces of unrecognised images first
everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway
# compute n=100 digits to display only
Xs = tf.gather(X, everything_incorrect_first)
Ys = tf.gather(Y, everything_incorrect_first)
Ys_ = tf.gather(Y_, everything_incorrect_first)
correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first)
digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left())
correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images
digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right())
computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images
#superimposed_digits = correct_tags+computed_tags
superimposed_digits = tf.where(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images
correct_bkg = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background
incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background
recognised_bkg = tf.gather(tf.concat([incorrect_bkg, correct_bkg], 0), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status
I = tf.image.grayscale_to_rgb(Xs)
I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first
I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
Islices = [] # 100 images => 10x10 image block
for imslice in range(lines):
Islices.append(tf.concat(tf.unstack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3])), 1))
I = tf.concat(Islices, 0)
return I
def inference_input():
"""Returns ops that convert raw image data to a 4D tensor representing a single image.
Taken from:
https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/inception_export.py
The input to the first op can be read using:
tf.gfile.FastGFile(image_filename, 'r').read()
"""
# Decode image into float range [0,1]
jpegs = tf.placeholder(tf.string, shape=(1), name='input')
image_buffer = tf.squeeze(jpegs, [0])
image = tf.image.decode_jpeg(image_buffer, channels=3)
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.image.central_crop(image, central_fraction=0.875)
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [FLAGS.image_size, FLAGS.image_size], align_corners=False)
image = tf.squeeze(image, [0])
# Rescale the image to [-1,-1]
image = tf.sub(image, 0.5)
image = tf.mul(image, 2.0)
images = tf.expand_dims(image, 0)
return images, jpegs
def _log_unnormalized_prob(self, x):
mean = tf.squeeze(tf.gather(x, [0], axis=-1), axis=-1)
precision = self._maybe_assert_valid_sample(
tf.squeeze(tf.gather(x, [1], axis=-1), axis=-1))
return (tf.math.xlogy(self.concentration - 0.5, precision)
- self.rate * precision
- 0.5 * self._lambda * precision * tf.square(mean - self.loc))
def preprocess_for_test(image, gt_boxes, gt_masks):
ih, iw = tf.shape(image)[0], tf.shape(image)[1]
## min size resizing
new_ih, new_iw = preprocess_utils._smallest_size_at_least(ih, iw, cfg.FLAGS.image_min_size)
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [new_ih, new_iw], align_corners=False)
image = tf.squeeze(image, axis=[0])
gt_masks = tf.expand_dims(gt_masks, -1)
gt_masks = tf.cast(gt_masks, tf.float32)
gt_masks = tf.image.resize_nearest_neighbor(gt_masks, [new_ih, new_iw], align_corners=False)
gt_masks = tf.cast(gt_masks, tf.int32)
gt_masks = tf.squeeze(gt_masks, axis=[-1])
scale_ratio = tf.to_float(new_ih) / tf.to_float(ih)
gt_boxes = preprocess_utils.resize_gt_boxes(gt_boxes, scale_ratio)
## zero mean image
image = tf.cast(image, tf.float32)
image = image / 256.0
image = (image - 0.5) * 2.0
image = tf.expand_dims(image, axis=0)
## rgb to bgr
image = tf.reverse(image, axis=[-1])
return image, gt_boxes, gt_masks
def _forward(self, obs_prob_list):
with tf.name_scope('init_scaling_factor'):
self.scale = tf.Variable(tf.zeros([self.N], tf.float64)) #scale factors
with tf.name_scope('forward_first_step'):
# initialize with state starting priors
init_prob = tf.mul(self.T0, tf.squeeze(obs_prob_list[0]))
# scaling factor at t=0
self.scale = tf.scatter_update(self.scale, 0, 1.0 / tf.reduce_sum(init_prob))
# scaled belief at t=0
self.forward = tf.scatter_update(self.forward, 0, self.scale[0] * init_prob)
# propagate belief
for step, obs_prob in enumerate(obs_prob_list[1:]):
with tf.name_scope('time_step-%s' %step):
# previous state probability
prev_prob = tf.expand_dims(self.forward[step, :], 0)
# transition prior
prior_prob = tf.matmul(prev_prob, self.T)
# forward belief propagation
forward_score = tf.mul(prior_prob, tf.squeeze(obs_prob))
forward_prob = tf.squeeze(forward_score)
# scaling factor
self.scale = tf.scatter_update(self.scale, step+1, 1.0 / tf.reduce_sum(forward_prob))
# Update forward matrix
self.forward = tf.scatter_update(self.forward, step+1, self.scale[step+1] * forward_prob)
def convolution(self, inputs, num_units):
x = tf.expand_dims(inputs, 3)
chan_in = 1
#Bigram
w_bigram = tf.get_variable("w_bigram", shape= [2,50,chan_in,num_units],
initializer= tf.contrib.layers.xavier_initializer_conv2d())
b_bigram = tf.get_variable("b_bigram", shape= [num_units])
y_bigram = self.nonlin(tf.nn.conv2d(x, w_bigram, strides= [1,1,1,1], padding='VALID') + b_bigram)
h_bigram = tf.reduce_max(tf.squeeze(y_bigram) , 1)
#Trigram
w_trigram = tf.get_variable("w_trigram", shape= [3,50,chan_in,num_units],
initializer= tf.contrib.layers.xavier_initializer_conv2d())
b_trigram = tf.get_variable("b_trigram", shape= [num_units])
y_trigram = self.nonlin(tf.nn.conv2d(x, w_trigram, strides= [1,1,1,1], padding='VALID') + b_trigram)
h_trigram = tf.reduce_max(tf.squeeze(y_trigram) , 1)
#Quin-gram
w_quingram = tf.get_variable("w_quingram", shape= [3,50,chan_in,num_units],
initializer= tf.contrib.layers.xavier_initializer_conv2d())
b_quingram = tf.get_variable("b_quingram", shape= [num_units])
y_quingram = self.nonlin(tf.nn.conv2d(x, w_trigram, strides= [1,1,1,1], padding='VALID') + b_trigram)
h_quingram = tf.reduce_max(tf.squeeze(y_quingram) , 1)
if self.hyperparams['conv_type'] == 'bigram':
h = h_bigram
elif self.hyperparams['conv_type'] == 'trigram':
h = h_trigram
elif self.hyperparams['conv_type'] == 'quingram':
h = h_quingram
elif self.hyperparams['conv_type'] == 'inception':
h = tf.concat(1, [h_bigram, h_trigram, h_quingram])
return h
def filename2image(image_size=299, central_fraction=0.875):
"""Returns ops that convert a filename to a 4D tensor representing a single image.
Usage:
image_op, buffer_op = decode_filename()
imbuffer = tf.gfile.FastGFile('path/to/file.jpg', 'r').read()
with tf.Session() as sess:
image = sess.run([image_op], feed_dict={buffer_op : [imbuffer]})
plt.imshow(image)
Taken from:
https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/inception_export.py
Args:
image_size (int): image is resized to this (default 299 for Inception v3)
central_fraction (float): the central fraction crop to take
Returns:
Two ops, one for the image buffer input, the other the tensorflow-ready image.
"""
image_buffer = tf.placeholder(tf.string, shape=(1))
image_buffer = tf.squeeze(image_buffer, [0])
image = tf.image.decode_jpeg(image_buffer, channels=3)
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.image.central_crop(image, central_fraction)
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [image_size, image_size], align_corners=False)
image = tf.squeeze(image, [0])
image = tf.sub(image, 0.5)
image = tf.mul(image, 2.0)
image = tf.expand_dims(image, 0)
return image, image_buffer
def attention(self,ref, query, with_softmax, scope="attention"):
"""
:param ref: encoder的输出
:param query: decoder的输出
:param with_softmax:
:param scope:
:return:
"""
with tf.variable_scope(scope):
W_1 = tf.get_variable("W_e", [self.hidden_dim, self.attention_dim], initializer=self.initializer) # L x A
W_2 = tf.get_variable("W_d", [self.hidden_dim, self.attention_dim], initializer=self.initializer) # L * A
dec_portion = tf.matmul(query, W_2)
scores = [] # S * B
v_blend = tf.get_variable("v_blend", [self.attention_dim, 1], initializer=self.initializer) # A x 1
bais_blend = tf.get_variable("bais_v_blend", [1], initializer=self.initializer) # 1 x 1
for i in range(self.max_enc_length + 1):
refi = tf.matmul(tf.squeeze(ref[:,i,:]),W_1)
ui = tf.add(tf.matmul(tf.nn.tanh(dec_portion+refi),v_blend),bais_blend) # B * 1
scores.append(tf.squeeze(ui))
scores = tf.transpose(scores,[1,0]) # B * S
if with_softmax:
return tf.nn.softmax(scores,dim=1)
else:
return scores
def compute_accuracy(x, l, mask):
"""Compute model accuracy."""
preds = ch_model.get_probs(x)
preds = tf.squeeze(preds)
preds = tf.argmax(preds, -1, output_type=l.dtype)
_, acc_update_op = tf.metrics.accuracy(l, preds, weights=mask)
if FLAGS.surrogate_attack:
preds = sur_ch_model.get_probs(x)
preds = tf.squeeze(preds)
preds = tf.argmax(preds, -1, output_type=l.dtype)
acc_update_op = tf.tuple((acc_update_op,
tf.metrics.accuracy(l, preds, weights=mask)[1]))
sess.run(tf.initialize_local_variables())
for i in range(FLAGS.eval_steps):
tf.logging.info(
"\tEvaluating batch [%d / %d]" % (i + 1, FLAGS.eval_steps))
acc = sess.run(acc_update_op)
if FLAGS.surrogate_attack:
tf.logging.info("\tFinal acc: (%.4f, %.4f)" % (acc[0], acc[1]))
else:
tf.logging.info("\tFinal acc: %.4f" % acc)
return acc
def hnet_transformation(gt_pts, transformation_coeffcient, name):
"""
:param gt_pts:
:param transformation_coeffcient:
:param name:
:return:
"""
with tf.variable_scope(name):
# 首先映射原始标签点对
transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1)
H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]])
H_shape = tf.constant([9])
H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape)
H = tf.reshape(H, shape=[3, 3])
gt_pts = tf.transpose(gt_pts)
pts_projects = tf.matmul(H, gt_pts)
# 求解最小二乘二阶多项式拟合参数矩阵
Y = tf.transpose(pts_projects[1, :])
X = tf.transpose(pts_projects[0, :])
Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32))
Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1)
w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)),
tf.transpose(Y_stack)),
tf.expand_dims(X, -1))
# 利用二阶多项式参数求解拟合位置
x_preds = tf.matmul(Y_stack, w)
preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1))
preds_fit = tf.stack([tf.squeeze(x_preds, -1), Y], axis=1)
x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds)
return x_transformation_back
def unpool_layer2x2(self, x, raveled_argmax, out_shape):
argmax = self.unravel_argmax(raveled_argmax, tf.to_int64(out_shape))
output = tf.zeros([out_shape[1], out_shape[2], out_shape[3]])
height = tf.shape(output)[0]
width = tf.shape(output)[1]
channels = tf.shape(output)[2]
t1 = tf.to_int64(tf.range(channels))
t1 = tf.tile(t1, [((width + 1) // 2) * ((height + 1) // 2)])
t1 = tf.reshape(t1, [-1, channels])
t1 = tf.transpose(t1, perm=[1, 0])
t1 = tf.reshape(t1, [channels, (height + 1) // 2, (width + 1) // 2, 1])
t2 = tf.squeeze(argmax)
t2 = tf.pack((t2[0], t2[1]), axis=0)
t2 = tf.transpose(t2, perm=[3, 1, 2, 0])
t = tf.concat(3, [t2, t1])
indices = tf.reshape(t, [((height + 1) // 2) * ((width + 1) // 2) * channels, 3])
x1 = tf.squeeze(x)
x1 = tf.reshape(x1, [-1, channels])
x1 = tf.transpose(x1, perm=[1, 0])
values = tf.reshape(x1, [-1])
delta = tf.SparseTensor(indices, values, tf.to_int64(tf.shape(output)))
return tf.expand_dims(tf.sparse_tensor_to_dense(tf.sparse_reorder(delta)), 0)
def randomly_scale_image_and_label(image, label=None, scale=1.0):
"""Randomly scales image and label.
Args:
image: Image with shape [height, width, 3].
label: Label with shape [height, width, 1].
scale: The value to scale image and label.
Returns:
Scaled image and label.
"""
# No random scaling if scale == 1.
if scale == 1.0:
return image, label
image_shape = tf.shape(image)
new_dim = tf.cast(
tf.cast([image_shape[0], image_shape[1]], tf.float32) * scale,
tf.int32)
# Need squeeze and expand_dims because image interpolation takes
# 4D tensors as input.
image = tf.squeeze(tf.image.resize_bilinear(
tf.expand_dims(image, 0),
new_dim,
align_corners=True), [0])
if label is not None:
label = tf.squeeze(tf.image.resize_nearest_neighbor(
tf.expand_dims(label, 0),
new_dim,
align_corners=True), [0])
return image, label
def build_detector(self):
img_size = self.config['image_size']
self.image_ph = tf.placeholder(shape=[None, None, 3],
dtype=tf.float32, name='img_ph')
self.seg_ph = tf.placeholder(shape=[None, None], dtype=tf.int32, name='seg_ph')
img = tf.image.resize_bilinear(tf.expand_dims(self.image_ph, 0),
(img_size, img_size))
self.net.create_trunk(img)
if args.detect:
self.net.create_multibox_head(self.loader.num_classes)
confidence = tf.nn.softmax(tf.squeeze(self.net.outputs['confidence']))
location = tf.squeeze(self.net.outputs['location'])
self.nms(location, confidence, self.bboxer.tiling)
if args.segment:
self.net.create_segmentation_head(self.loader.num_classes)
self.segmentation = self.net.outputs['segmentation']
seg_shape = tf.shape(self.image_ph)[:2]
self.segmentation = tf.image.resize_bilinear(self.segmentation, seg_shape)
self.segmentation = tf.cast(tf.argmax(tf.squeeze(self.segmentation), axis=-1), tf.int32)
self.segmentation = tf.reshape(self.segmentation, seg_shape)
self.segmentation.set_shape([None, None])
if not self.no_gt:
easy_mask = self.seg_ph <= self.loader.num_classes
predictions = tf.boolean_mask(self.segmentation, easy_mask)
labels = tf.boolean_mask(self.seg_ph, easy_mask)
self.mean_iou, self.iou_update = mean_iou(predictions, labels, self.loader.num_classes)
else:
self.mean_iou = tf.constant(0)
self.iou_update = tf.constant(0)
def buildConvolution(self):
q_embedding = self.tensors['q_embedding']
a_embedding = self.tensors['a_embedding']
with tf.name_scope('convolution'):
filter_shape = (self.params['filters'][0], self.wdim, 1, self.params['nb_filter'])
W = glorot_normal(filter_shape, name="W")
b = tf.Variable(tf.constant(0.0, shape=(self.params['nb_filter'],)), name="b")
q_conv = tf.nn.conv2d(
tf.expand_dims(q_embedding, -1),
W,
strides=[1,1,1,1],
padding="VALID",
name="q_conv"
)
a_conv = tf.nn.conv2d(
tf.expand_dims(a_embedding, -1),
W,
strides=[1,1,1,1],
padding="VALID",
name = "a_conv"
)
q_conv = tf.squeeze(q_conv, [2])
a_conv = tf.squeeze(a_conv, [2])
# shape = (batch, q_length, NUM_FILTERS)
q_relu = tf.nn.relu(tf.nn.bias_add(q_conv, b), name="q_relu")
# shape = (batch, a_length, NUM_FILTERS)
a_relu = tf.nn.relu(tf.nn.bias_add(a_conv, b), name="q_relu")
self.tensors['q_conv'] = q_conv
self.tensors['a_conv'] = a_conv
self.tensors['q_relu'] = q_relu
self.tensors['a_relu'] = a_relu
self.tensors.setdefault('weights', []).append(b)
self.tensors.setdefault('summary', []).append(tf.nn.zero_fraction(a_relu))
def _match(self, similarity_matrix, valid_rows):
"""Bipartite matches a collection rows and columns. A greedy bi-partite.
TODO(rathodv): Add num_valid_columns options to match only that many columns
with all the rows.
Args:
similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
where higher values mean more similar.
valid_rows: A boolean tensor of shape [N] indicating the rows that are
valid.
Returns:
match_results: int32 tensor of shape [M] with match_results[i]=-1
meaning that column i is not matched and otherwise that it is matched to
row match_results[i].
"""
valid_row_sim_matrix = tf.gather(similarity_matrix,
tf.squeeze(tf.where(valid_rows), axis=-1))
invalid_row_sim_matrix = tf.gather(
similarity_matrix,
tf.squeeze(tf.where(tf.logical_not(valid_rows)), axis=-1))
similarity_matrix = tf.concat(
[valid_row_sim_matrix, invalid_row_sim_matrix], axis=0)
# Convert similarity matrix to distance matrix as tf.image.bipartite tries
# to find minimum distance matches.
distance_matrix = -1 * similarity_matrix
num_valid_rows = tf.reduce_sum(tf.to_float(valid_rows))
_, match_results = image_ops.bipartite_match(
distance_matrix, num_valid_rows=num_valid_rows)
match_results = tf.reshape(match_results, [-1])
match_results = tf.cast(match_results, tf.int32)
return match_results
def _add_box_predictions_to_feature_maps(self, feature_maps):
"""Adds box predictors to each feature map and returns concatenated results.
Args:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i]
Returns:
box_encodings: 4-D float tensor of shape [batch_size, num_anchors,
box_code_dimension] containing predicted boxes.
class_predictions_with_background: 2-D float tensor of shape
[batch_size, num_anchors, num_classes+1] containing class predictions
(logits) for each of the anchors. Note that this tensor *includes*
background class predictions (at class index 0).
Raises:
RuntimeError: if the number of feature maps extracted via the
extract_features method does not match the length of the
num_anchors_per_locations list that was passed to the constructor.
RuntimeError: if box_encodings from the box_predictor does not have
shape of the form [batch_size, num_anchors, 1, code_size].
"""
num_anchors_per_location_list = 1
if len(feature_maps) != len(num_anchors_per_location_list):
raise RuntimeError('the number of feature maps must match the '
'length of self.anchors.NumAnchorsPerLocation().')
box_encodings_list = []
mask_encodings_list = []
for idx, (feature_map, num_anchors_per_location
) in enumerate(zip(feature_maps, num_anchors_per_location_list)):
box_predictor_scope = 'BoxPredictor_{}'.format(idx)
box_predictions = self._box_predictor.predict(feature_map,
num_anchors_per_location,
box_predictor_scope)
box_encodings = box_predictions[bpredictor.BOX_ENCODINGS]
mask_encodings = box_predictions[bpredictor.MASK_PREDICTIONS]
box_encodings_shape = box_encodings.get_shape().as_list()
if len(box_encodings_shape) != 5 or box_encodings_shape[2] != 1:
raise RuntimeError('box_encodings from the box_predictor must be of '
'shape `[batch_size, num_anchors, 1, code_size]`; '
'actual shape', box_encodings_shape)
box_encodings = tf.squeeze(box_encodings, axis=2)
mask_encodings = tf.squeeze(mask_encodings, axis=2)
box_encodings_list.append(box_encodings)
mask_encodings_list.append(mask_encodings)
"""
num_predictions = sum(
[tf.shape(box_encodings)[1] for box_encodings in box_encodings_list])
num_anchors = self.anchors.num_boxes()
anchors_assert = tf.assert_equal(num_anchors, num_predictions, [
'Mismatch: number of anchors vs number of predictions', num_anchors,
num_predictions
])
"""
box_encodings = tf.concat(box_encodings_list, 1)
mask_encodings = tf.concat(mask_encodings_list, 1)
return box_encodings, mask_encodings
def build_network(self):
net_tensors = self.net_tensors
with self.net_graph.as_default(), tf.device(self.net_device):
logits = tf.placeholder(dtype=tf.float32, shape=(self.batch_size, self.image_classes))
labels = tf.placeholder(dtype=tf.int32, shape=(self.batch_size,))
lambs = tf.placeholder(dtype=tf.float32, shape=(self.image_classes,))
# put a sigfunction on logits and then transpose
logits = tf.transpose(framwork.sig_func(logits))
# according to the labels, erase rows which is not in labels
labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32)
labels_num = self.image_classes
logits = tf.gather(logits, indices=labels_unique)
lambs = tf.gather(lambs, indices=labels_unique)
# set the value of each row to True when it occurs in labels
templete = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size])
labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1])
indict_logic = tf.equal(labels_expand, templete)
# split the tensor along rows
logit_list = tf.split(0, labels_num, logits)
indict_logic_list = tf.split(0, labels_num, indict_logic)
lamb_list = tf.split(0, self.image_classes, lambs)
logit_list = [tf.squeeze(item) for item in logit_list]
indict_logic_list = [tf.squeeze(item) for item in indict_logic_list]
left_right_tuples = list()
for i in range(self.image_classes):
left_right_tuples.append(framwork.lamb_func(logit_list[i], indict_logic_list[i], lamb=lamb_list[i]))
# func = framwork.lamb_func()
# left_right_tuples = map(func, logit_list, indict_logic_list, lamb_list)
net_tensors.update({'left_right_tuples': left_right_tuples, 'logits': logits, 'labels': labels,
'lambs': lambs})
def conpute_loss(scores, target):
""" Compute the perplexity of the batch
Args:
scores: 3D tensor, shape=(BATCH_SIZE, 1, S_FRENCH, T_FRENCH)
target: 4D tensor, shape=(BATCH_SIZE, 1, S_FRENCH, T_FRENCH)
Returns:
tf.float32 tensor
"""
with tf.name_scope('Loss_computation'):
sortie_loss = tf.squeeze(target)
scores = tf.squeeze(scores)
loss = tf.reduce_sum(tf.mul(scores, sortie_loss), reduction_indices=2) # Get the activation of the target token
#loss = tf.reduce_sum(loss,reduction_indices=2)
loss = tf.clip_by_value(loss, clip_value_min=1e-10, clip_value_max=1.0)
#loss =
loss = tf.reduce_sum(tf.log(loss), reduction_indices=1)
loss = -tf.reduce_mean(loss)
l2_weights = 0.00
with tf.variable_scope('Embeddings', reuse=True):
w = tf.get_variable('weights')
b = tf.get_variable('biases')
loss = loss + l2_weights*tf.nn.l2_loss(w) + l2_weights*tf.nn.l2_loss(b)
with tf.variable_scope('Decoding', reuse=True):
w = tf.get_variable('weights')
b = tf.get_variable('biases')
loss = loss + l2_weights*tf.nn.l2_loss(w) + l2_weights*tf.nn.l2_loss(b)
return loss
def get_content_based_attention_vectors(query_vectors):
'''
function that returns the alpha_content vector using the yt-1 (query vectors)
'''
# use the W_f and b_f to transform the query_vectors to the shape of f_values
f_trans_query_vectors = tf.matmul(W_f, tf.transpose(query_vectors)) + b_f
# use the W_c and b_c to transform the query_vectors to the shape of h_values
h_trans_query_vectors = tf.matmul(W_c, tf.transpose(query_vectors)) + b_c
# transpose and expand the dims of the f_trans_query_vectors
f_trans_query_matrices = tf.expand_dims(tf.transpose(f_trans_query_vectors), axis=-1)
# obtain the field attention_values by using the matmul operation
field_attention_values = tf.matmul(tf_field_embedded, f_trans_query_matrices)
# perform the same process for the h_trans_query_vectors
h_trans_query_matrices = tf.expand_dims(tf.transpose(h_trans_query_vectors), axis=-1)
hidden_attention_values = tf.matmul(encoded_input, h_trans_query_matrices)
# drop the last dimension (1 sized)
field_attention_values = tf.squeeze(field_attention_values, axis=[-1])
hidden_attention_values = tf.squeeze(hidden_attention_values, axis=[-1])
# free up non_required resources:
ret_value = tf.nn.softmax(field_attention_values * hidden_attention_values, name="softmax")
# return the element wise multiplied values followed by softmax
return ret_value
def transform_input_data(tensor_dict,
model_preprocess_fn,
image_resizer_fn,
num_classes,
data_augmentation_fn=None,
merge_multiple_boxes=False,
retain_original_image=False,
use_multiclass_scores=False,
use_bfloat16=False):
"""A single function that is responsible for all input data transformations.
Data transformation functions are applied in the following order.
1. If key fields.InputDataFields.image_additional_channels is present in
tensor_dict, the additional channels will be merged into
fields.InputDataFields.image.
2. data_augmentation_fn (optional): applied on tensor_dict.
3. model_preprocess_fn: applied only on image tensor in tensor_dict.
4. image_resizer_fn: applied on original image and instance mask tensor in
tensor_dict.
5. one_hot_encoding: applied to classes tensor in tensor_dict.
6. merge_multiple_boxes (optional): when groundtruth boxes are exactly the
same they can be merged into a single box with an associated k-hot class
label.
Args:
tensor_dict: dictionary containing input tensors keyed by
fields.InputDataFields.
model_preprocess_fn: model's preprocess function to apply on image tensor.
This function must take in a 4-D float tensor and return a 4-D preprocess
float tensor and a tensor containing the true image shape.
image_resizer_fn: image resizer function to apply on groundtruth instance
`masks. This function must take a 3-D float tensor of an image and a 3-D
tensor of instance masks and return a resized version of these along with
the true shapes.
num_classes: number of max classes to one-hot (or k-hot) encode the class
labels.
data_augmentation_fn: (optional) data augmentation function to apply on
input `tensor_dict`.
merge_multiple_boxes: (optional) whether to merge multiple groundtruth boxes
and classes for a given image if the boxes are exactly the same.
retain_original_image: (optional) whether to retain original image in the
output dictionary.
use_multiclass_scores: whether to use multiclass scores as
class targets instead of one-hot encoding of `groundtruth_classes`.
use_bfloat16: (optional) a bool, whether to use bfloat16 in training.
Returns:
A dictionary keyed by fields.InputDataFields containing the tensors obtained
after applying all the transformations.
"""
# Reshape flattened multiclass scores tensor into a 2D tensor of shape
# [num_boxes, num_classes].
if fields.InputDataFields.multiclass_scores in tensor_dict:
tensor_dict[fields.InputDataFields.multiclass_scores] = tf.reshape(
tensor_dict[fields.InputDataFields.multiclass_scores], [
tf.shape(tensor_dict[fields.InputDataFields.groundtruth_boxes])[0],
num_classes
])
if fields.InputDataFields.groundtruth_boxes in tensor_dict:
tensor_dict = util_ops.filter_groundtruth_with_nan_box_coordinates(
tensor_dict)
tensor_dict = util_ops.filter_unrecognized_classes(tensor_dict)
if retain_original_image:
tensor_dict[fields.InputDataFields.original_image] = tf.cast(
image_resizer_fn(tensor_dict[fields.InputDataFields.image], None)[0],
tf.uint8)
if fields.InputDataFields.image_additional_channels in tensor_dict:
channels = tensor_dict[fields.InputDataFields.image_additional_channels]
tensor_dict[fields.InputDataFields.image] = tf.concat(
[tensor_dict[fields.InputDataFields.image], channels], axis=2)
# Apply data augmentation ops.
if data_augmentation_fn is not None:
tensor_dict = data_augmentation_fn(tensor_dict)
# Apply model preprocessing ops and resize instance masks.
image = tensor_dict[fields.InputDataFields.image]
preprocessed_resized_image, true_image_shape = model_preprocess_fn(
tf.expand_dims(tf.to_float(image), axis=0))
if use_bfloat16:
preprocessed_resized_image = tf.cast(
preprocessed_resized_image, tf.bfloat16)
tensor_dict[fields.InputDataFields.image] = tf.squeeze(
preprocessed_resized_image, axis=0)
tensor_dict[fields.InputDataFields.true_image_shape] = tf.squeeze(
true_image_shape, axis=0)
if fields.InputDataFields.groundtruth_instance_masks in tensor_dict:
masks = tensor_dict[fields.InputDataFields.groundtruth_instance_masks]
_, resized_masks, _ = image_resizer_fn(image, masks)
if use_bfloat16:
resized_masks = tf.cast(resized_masks, tf.bfloat16)
tensor_dict[fields.InputDataFields.
groundtruth_instance_masks] = resized_masks
# Transform groundtruth classes to one hot encodings.
label_offset = 1
zero_indexed_groundtruth_classes = tensor_dict[
fields.InputDataFields.groundtruth_classes] - label_offset
tensor_dict[fields.InputDataFields.groundtruth_classes] = tf.one_hot(
zero_indexed_groundtruth_classes, num_classes)
if use_multiclass_scores:
tensor_dict[fields.InputDataFields.groundtruth_classes] = tensor_dict[
fields.InputDataFields.multiclass_scores]
tensor_dict.pop(fields.InputDataFields.multiclass_scores, None)
if fields.InputDataFields.groundtruth_confidences in tensor_dict:
groundtruth_confidences = tensor_dict[
fields.InputDataFields.groundtruth_confidences]
# Map the confidences to the one-hot encoding of classes
tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
tf.reshape(groundtruth_confidences, [-1, 1]) *
tensor_dict[fields.InputDataFields.groundtruth_classes])
else:
groundtruth_confidences = tf.ones_like(
zero_indexed_groundtruth_classes, dtype=tf.float32)
tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
tensor_dict[fields.InputDataFields.groundtruth_classes])
if merge_multiple_boxes:
merged_boxes, merged_classes, merged_confidences, _ = (
util_ops.merge_boxes_with_multiple_labels(
tensor_dict[fields.InputDataFields.groundtruth_boxes],
zero_indexed_groundtruth_classes,
groundtruth_confidences,
num_classes))
merged_classes = tf.cast(merged_classes, tf.float32)
tensor_dict[fields.InputDataFields.groundtruth_boxes] = merged_boxes
tensor_dict[fields.InputDataFields.groundtruth_classes] = merged_classes
tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
merged_confidences)
if fields.InputDataFields.groundtruth_boxes in tensor_dict:
tensor_dict[fields.InputDataFields.num_groundtruth_boxes] = tf.shape(
tensor_dict[fields.InputDataFields.groundtruth_boxes])[0]
return tensor_dict
def __init__(self, args, infer=False):
self.args = args
if infer:
args.batch_size = 1
args.seq_length = 1
if args.model == 'rnn':
cell_fn = rnn.BasicRNNCell
elif args.model == 'gru':
cell_fn = rnn.GRUCell
elif args.model == 'lstm':
cell_fn = rnn.BasicLSTMCell
else:
raise Exception("model type not supported: {}".format(args.model))
cells = []
for _ in range(args.num_layers):
cell = cell_fn(args.rnn_size)
cells.append(cell)
self.cell = cell = rnn.MultiRNNCell(cells)
self.input_data = tf.placeholder(tf.int32,
[args.batch_size, args.seq_length])
self.targets = tf.placeholder(tf.int32,
[args.batch_size, args.seq_length])
self.initial_state = cell.zero_state(args.batch_size, tf.float32)
self.batch_pointer = tf.Variable(0,
name="batch_pointer",
trainable=False,
dtype=tf.int32)
self.inc_batch_pointer_op = tf.assign(self.batch_pointer,
self.batch_pointer + 1)
self.epoch_pointer = tf.Variable(0,
name="epoch_pointer",
trainable=False)
self.batch_time = tf.Variable(0.0, name="batch_time", trainable=False)
tf.summary.scalar("time_batch", self.batch_time)
def variable_summaries(var):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
#with tf.name_scope('stddev'):
# stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
#tf.summary.scalar('stddev', stddev)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
#tf.summary.histogram('histogram', var)
with tf.variable_scope('rnnlm'):
softmax_w = tf.get_variable("softmax_w",
[args.rnn_size, args.vocab_size])
variable_summaries(softmax_w)
softmax_b = tf.get_variable("softmax_b", [args.vocab_size])
variable_summaries(softmax_b)
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding",
[args.vocab_size, args.rnn_size])
inputs = tf.split(
tf.nn.embedding_lookup(embedding, self.input_data),
args.seq_length, 1)
inputs = [tf.squeeze(input_, [1]) for input_ in inputs]
def loop(prev, _):
prev = tf.matmul(prev, softmax_w) + softmax_b
prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))
return tf.nn.embedding_lookup(embedding, prev_symbol)
outputs, last_state = legacy_seq2seq.rnn_decoder(
inputs,
self.initial_state,
cell,
loop_function=loop if infer else None,
scope='rnnlm')
output = tf.reshape(tf.concat(outputs, 1), [-1, args.rnn_size])
self.logits = tf.matmul(output, softmax_w) + softmax_b
self.probs = tf.nn.softmax(self.logits)
loss = legacy_seq2seq.sequence_loss_by_example(
[self.logits], [tf.reshape(self.targets, [-1])],
[tf.ones([args.batch_size * args.seq_length])], args.vocab_size)
self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length
tf.summary.scalar("cost", self.cost)
self.final_state = last_state
self.lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars),
args.grad_clip)
optimizer = tf.train.AdamOptimizer(self.lr)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def test_classifier(classifier_model_dir: pathlib.Path,
fwd_model_dir: List[pathlib.Path],
n_actions: int,
saved_state: Optional[pathlib.Path],
generate_actions: Callable):
fwd_model, _ = dynamics_utils.load_generic_model([pathlib.Path(p) for p in fwd_model_dir])
classifier: NNClassifierWrapper = classifier_utils.load_generic_model([classifier_model_dir])
service_provider = GazeboServices()
service_provider.setup_env(verbose=0,
real_time_rate=0,
max_step_size=fwd_model.data_collection_params['max_step_size'],
play=False)
if saved_state:
service_provider.restore_from_bag(saved_state)
scenario = fwd_model.scenario
scenario.on_before_get_state_or_execute_action()
# NOTE: perhaps it would make sense to have a "fwd_model" have API for get_env, get_state, sample_action, etc
# because the fwd_model knows it's scenario, and importantly it also knows it's data_collection_params
# which is what we're using here to pass to the scenario methods
params = fwd_model.data_collection_params
environment = numpify(scenario.get_environment(params))
start_state = numpify(scenario.get_state())
start_state_tiled = repeat(start_state, n_actions, axis=0, new_axis=True)
start_states_tiled = add_time_dim(start_state_tiled)
actions = generate_actions(environment, start_state_tiled, scenario, params, n_actions)
environment_tiled = repeat(environment, n_actions, axis=0, new_axis=True)
actions_dict = sequence_of_dicts_to_dict_of_tensors(actions)
actions_dict = add_time_dim(actions_dict)
predictions, _ = fwd_model.propagate_differentiable_batched(environment=environment_tiled,
state=start_states_tiled,
actions=actions_dict)
# Run classifier
state_sequence_length = 2
accept_probabilities, _ = classifier.check_constraint_batched_tf(environment=environment_tiled,
predictions=predictions,
actions=actions_dict,
state_sequence_length=state_sequence_length,
batch_size=n_actions)
# animate over the sampled actions
anim = RvizAnimation(scenario=scenario,
n_time_steps=n_actions,
init_funcs=[init_viz_env],
t_funcs=[
lambda s, e, t: init_viz_env(s, e),
viz_transition_for_model_t({}, fwd_model),
ExperimentScenario.plot_accept_probability_t,
],
)
example = {
'accept_probability': tf.squeeze(accept_probabilities, axis=1),
}
example.update(environment)
example.update(predictions)
example.update(actions_dict)
anim.play(example)
def squeeze(x, axis):
'''Removes a 1-dimension from the tensor at index "axis".
'''
return tf.squeeze(x, [axis])
def get_model(v,
displacement,
edges_ss,
edges_ss_n,
is_training,
final_activation,
bn_decay=None):
"""Generat mesh network
:param v: variance normalized source vertices
:param displacement: displacement of each vertex from variance normalized source to variance normalized target
:param edges_ss: the ij vertix, to compute w_ij, specified for the source vertices
:param edges_ss_n: number of edges
:param is_training: need to check what it does, exactly.
:param final_activation: what activation to put in the final layer that drives w_ij, if range to be limited this hsould be
sigmoidal.
:param bn_decay: need to check what it does, exactly.
:return: neural network that produces w_ij to be later assambled into a hessian.
"""
concat_source_displacement = tf.concat([v, displacement], 2)
batch_size = v.get_shape()[0].value
num_point = v.get_shape()[1].value
input_image = tf.expand_dims(concat_source_displacement, -1)
net = tf_util.conv2d(input_image,
64, [1, 4],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
# net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
# scope='dp1')
global_feat = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
global_feat_expand = tf.tile(global_feat, [1, edges_ss_n, 1, 1])
edges_expanded = tf.expand_dims(edges_ss, axis=2)
concat_feat = tf.concat([edges_expanded, global_feat_expand], 3)
net = tf_util.conv2d(concat_feat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv8',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv9',
bn_decay=bn_decay)
dim = 2
net = tf_util.conv2d(net,
dim, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=final_activation,
is_training=is_training,
scope='conv10')
w_ij = tf.squeeze(net, [2]) # BxNxC
return w_ij
def mesh_get_loss(w_ij,
deng_x_at_x0_sub,
eng_at_x0_sub,
eng_at_xnew_sub,
d_n,
alpha,
epsilon,
BATCH=1,
nv_sub=12):
"""s : Source and target patch size on which we test the energy (1-ring) as
opposed to a slightly larger patch that the networks sees.
(hessian will be "BATCH x S x S x DATA_DIM", currently data dim = 1);
w_ij : output of all the weight in hessian. Each w_ij is driven by the same network
replicated for each edge. So a w_ij is a function of source patch, target
patch and an edge. There are (S *(S-1))/2 edges in patch of size S,
leading to all w_ij being a vector of size "BATCH x (S *(S-1))/2 x DATA_DIM"";
gard : gradient with respect to target of E_{source} @ target.
Vector of size "BATCH x S x DATA_DIM";
epsilon : noise added to gradient size "BATCH x S x DATA_DIM";
alpha : step size "BATCH x 1";
energy_true : Energy value E_{source} @ (target + alpha grad + epsilon) "BATCH x 1".
TODO: currently support DATA_DIM = 1"""
end_points = {}
# update hessian with weights where the weight go to the indexes
# specified by all_indixing_with_batch array.
wx_ij_flat = tf.reshape(w_ij[:, :, 0], [-1])
wy_ij_flat = tf.reshape(w_ij[:, :, 1], [-1])
Hx_final, Hy_final = assemble_h(wx_ij_flat, wy_ij_flat)
# compute quadratic form
v = tf.reshape(alpha, (BATCH, 1, 1)) * d_n + epsilon
vx = tf.reshape(v[:, :, 0], (BATCH, nv_sub, 1))
vy = tf.reshape(v[:, :, 1], (BATCH, nv_sub, 1))
# compute quadratic form v^top Q v
Qx = tf.matmul(a=tf.matmul(a=vx,
b=tf.cast(Hx_final, tf.float32),
transpose_a=True),
b=vx)
Qx = tf.squeeze(Qx, [2])
Qy = tf.matmul(a=tf.matmul(a=vy,
b=tf.cast(Hy_final, tf.float32),
transpose_a=True),
b=vy)
Qy = tf.squeeze(Qy, [2])
# remove the linear part
v_flat = tf.reshape(v, [BATCH, -1, 1])
denergy_dx_flat = tf.reshape(deng_x_at_x0_sub, [BATCH, -1, 1])
# normalized direction times alpha: v_flat gradient, but gradient is not normalized
tmp = tf.matmul(a=v_flat, b=denergy_dx_flat, transpose_a=True)
energy_lin_removed = eng_at_xnew_sub - (eng_at_x0_sub -
tf.squeeze(tmp, [2]))
energy_dif = energy_lin_removed - Qx - Qy
end_points['eng_at_xnew_sub'] = eng_at_xnew_sub
end_points['energy_dif'] = energy_dif
end_points['Qx'] = Qx
end_points['Qy'] = Qy
end_points['energy_lin_removed'] = energy_lin_removed
end_points['Hx_final'] = Hx_final
end_points['Hy_final'] = Hy_final
energy_dif_loss = tf.nn.l2_loss(energy_dif)
tf.summary.scalar('energy_dif_loss', energy_dif_loss)
return (energy_dif_loss, end_points)
radius = 1.7
shift = np.asarray((0.5, 0.1, 0.5)) #shift from voxelization
center = np.array([0, 0.4, 0]) + shift
angle = np.pi/2 - i * 2 * np.pi / img_num
pos = np.array([radius * np.sin(angle), 0.5, radius * np.cos(angle)]) + shift
mv = pyrr.matrix44.create_look_at(eye=pos, target=center, up=np.array([0, 1, 0])).transpose()
matrix = np.matmul(p, mv)
matrix = np.linalg.inv(matrix)
tf.keras.backend.set_learning_phase(False)
img = generator([model, matrix])
img = img * 0.5 + 0.5
img = tf.squeeze(img)
img = tf.image.convert_image_dtype(img, dtype=tf.uint8, saturate=True)
images.append(np.array(img)) # for gif
if not os.path.isdir(join(log_dir, name)):
os.mkdir(join(log_dir, name))
img = tf.image.encode_png(img)
tf.write_file(join(log_dir, name, str(i).zfill(3)) + ".png", img)
if build_gif:
imageio.mimsave(join(log_dir, name + ".gif"), images)
def _distance_multihead(self, item_seq, user_latent_factor, item_id):
# horizontal conv layer
lengths = [i + 1 for i in range(self.L)]
out, out_h, out_v = None, None, None
# item_latent_factor = self.add_timing_signal(item_latent_factor)
item_latent_factor = tf.nn.embedding_lookup(self.Q, item_seq)
item_latent_factor_2 = tf.nn.embedding_lookup(self.X, item_seq)
query = key = self.add_timing_signal(item_latent_factor)
out = self.multihead_attention(queries=query,
keys=key,
value=item_latent_factor,
reuse=tf.AUTO_REUSE)
out = tf.reduce_mean(out, 1)
query_2 = key_2 = out
query_2 = tf.layers.dense(
inputs=query_2,
name="linear_project1",
units=self.num_factor,
activation=None,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate))
key_2 = tf.layers.dense(
inputs=key_2,
name="linear_project1",
units=self.num_factor,
activation=None,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate))
# value = tf.layers.dense(inputs= key, name="linear_project", units = seq_len_item, activation = None, kernel_initializer=tf.random_normal_initializer, reuse=True)
# b = tf.Variable(tf.random_normal([seq_len_user], stddev=1))
logits_2 = tf.matmul(query_2, key_2, transpose_b=True) / np.sqrt(
self.num_factor)
weights_2 = tf.nn.softmax(logits_2, name="attention_weights1")
mask_2 = tf.ones([self.L, self.L])
mask_2 = tf.matrix_set_diag(mask_2, tf.zeros([self.L]))
weights_2 = weights_2 * mask_2
out_2 = tf.reduce_mean(tf.matmul(weights_2, out), 1)
print("--------------")
print(np.shape(out))
print(np.shape(out_2))
w_items = tf.nn.embedding_lookup(self.X, item_id)
w_items_2 = tf.nn.embedding_lookup(self.Q, item_id)
b_items = tf.nn.embedding_lookup(self.b, item_id)
item_specific_bias = tf.nn.embedding_lookup(self.X, item_id)
x_tmp = []
for i in range(np.shape(w_items)[1]):
x_tmp.append(out)
x = tf.stack(x_tmp)
print(np.shape(x))
print(np.shape(w_items))
x = tf.transpose(x, [1, 0, 2])
u_tmp = []
for i in range(np.shape(w_items)[1]):
u_tmp.append(user_latent_factor)
u = tf.stack(u_tmp)
print(np.shape(u))
u = tf.transpose(u, [1, 0, 2])
# res = tf.reduce_sum(tf.multiply(x, w_items), 2) + b_items
res = 0.2 * tf.reduce_sum(
tf.square(w_items - u), 2) + 0.8 * tf.reduce_sum(
tf.square(x - w_items_2),
2) # + 0.1 * tf.reduce_sum(tf.square(x - u), 2)
print(np.shape(res))
return tf.squeeze(res)
def tf_predict(self, states, internals, update):
embedding = self.network.apply(x=states, internals=internals, update=update)
prediction = self.linear.apply(x=embedding)
return tf.squeeze(input=prediction, axis=1)
def inference(self, inputs, is_training):
d_out = self.config.d_out
feature = inputs['features']
feature = tf.layers.dense(feature, 8, activation=None, name='fc0')
feature = tf.nn.leaky_relu(
tf.layers.batch_normalization(feature,
-1,
0.99,
1e-6,
training=is_training))
feature = tf.expand_dims(feature, axis=2)
# ###########################Encoder############################
f_encoder_list = []
for i in range(self.config.num_layers):
f_encoder_i = self.dilated_res_block(feature, inputs['xyz'][i],
inputs['neigh_idx'][i],
d_out[i],
'Encoder_layer_' + str(i),
is_training)
f_sampled_i = self.random_sample(f_encoder_i, inputs['sub_idx'][i])
feature = f_sampled_i
# if i == 0:
# self.encoder_features = f_encoder_i
# self.encoder_sub_indices = inputs['sub_idx'][i]
if i == 0:
f_encoder_list.append(f_encoder_i)
self.encoder_features.append(f_encoder_i)
f_encoder_list.append(f_sampled_i)
self.encoder_features.append(f_sampled_i)
self.encoder_sub_indices.append(inputs['sub_idx'][i])
# ###########################Encoder############################
feature = helper_tf_util.conv2d(
f_encoder_list[-1], f_encoder_list[-1].get_shape()[3].value,
[1, 1], 'decoder_0', [1, 1], 'VALID', True, is_training)
# ###########################Decoder############################
f_decoder_list = []
for j in range(self.config.num_layers):
f_interp_i = self.nearest_interpolation(
feature, inputs['interp_idx'][-j - 1])
f_decoder_i = helper_tf_util.conv2d_transpose(
tf.concat([f_encoder_list[-j - 2], f_interp_i], axis=3),
f_encoder_list[-j - 2].get_shape()[-1].value, [1, 1],
'Decoder_layer_' + str(j), [1, 1],
'VALID',
bn=True,
is_training=is_training)
feature = f_decoder_i
f_decoder_list.append(f_decoder_i)
self.decoder_features.append(f_decoder_i)
self.decoder_sub_indices.append(inputs['interp_idx'][-j - 1])
# ###########################Decoder############################
f_layer_fc1 = helper_tf_util.conv2d(f_decoder_list[-1], 64, [1, 1],
'fc1', [1, 1], 'VALID', True,
is_training)
f_layer_fc2 = helper_tf_util.conv2d(f_layer_fc1, 32, [1, 1], 'fc2',
[1, 1], 'VALID', True, is_training)
f_layer_drop = helper_tf_util.dropout(f_layer_fc2,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
f_layer_fc3 = helper_tf_util.conv2d(f_layer_drop,
self.config.num_classes, [1, 1],
'fc', [1, 1],
'VALID',
False,
is_training,
activation_fn=None)
f_out = tf.squeeze(f_layer_fc3, [2])
return f_out
def main():
"""Create the model and start the training."""
start = time.time()
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# Predictions.
raw_output = net.layers['fc1_voc12']
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [-1, ])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, axis=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable): (len(conv_trainable) + len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
losses = []
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0:
loss_value, images, labels, preds, _ = sess.run([reduced_loss, image_batch, label_batch, pred, train_op],
feed_dict=feed_dict)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
losses.append(loss_value)
plotLoss(losses)
coord.request_stop()
coord.join(threads)
end = time.time()
print("Duration taken to complete the training process is {} seconds.".format(end - start))
savingParams = []
savingParams.append(loss_value)
savingParams.append(end - start)
# write saved parameters from training
with open(savingOutputDir, 'w') as f:
print(savingParams, file=f)
def _distance_self_attention(self,
item_seq,
user_id,
item_id,
isTrain=True):
# horizontal conv layer
lengths = [i + 1 for i in range(self.L)]
out, out_h, out_v = None, None, None
# print(np.shape(item_seq)[1])
# item_latent_factor = self.add_timing_signal(item_latent_factor)
item_latent_factor = tf.nn.embedding_lookup(
self.Q, item_seq) #+ self.user_specific_bias
item_latent_factor_2 = tf.nn.embedding_lookup(self.X, item_seq)
query = key = value = self.add_timing_signal(item_latent_factor)
if isTrain:
query = tf.layers.dense(
inputs=query,
name="linear_project",
units=self.num_factor,
activation=tf.nn.relu,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate))
#query = tf.layers.dropout(query, rate=0.0)
key = tf.layers.dense(
inputs=key,
name="linear_project",
units=self.num_factor,
activation=tf.nn.relu,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate),
)
#key = tf.layers.dropout(key, rate=0.3)
else:
query = tf.layers.dense(
inputs=query,
name="linear_project",
units=self.num_factor,
activation=tf.nn.relu,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate))
key = tf.layers.dense(
inputs=key,
name="linear_project",
units=self.num_factor,
activation=tf.nn.relu,
use_bias=False,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
reuse=tf.AUTO_REUSE,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=self.reg_rate))
logits = tf.matmul(query, key, transpose_b=True) / np.sqrt(
self.num_factor)
print(np.shape(logits))
weights = tf.nn.softmax(logits, dim=-1, name="attention_weights")
mask = tf.ones([self.L, self.L])
mask = tf.matrix_set_diag(mask, tf.zeros([self.L]))
weights = weights * mask
out = tf.matmul(weights, item_latent_factor)
self.weights = weights
print(np.shape(item_latent_factor))
self.out = tf.reduce_mean(out, 1)
print(np.shape(self.out))
w_items = tf.nn.embedding_lookup(self.X, item_id)
w_items_2 = tf.nn.embedding_lookup(self.Q, item_id)
w_items_3 = tf.nn.embedding_lookup(
self.V, user_id) #tf.nn.embedding_lookup(self.A, item_id)
self.bias_item = tf.nn.embedding_lookup(self.b, item_id)
x_tmp = []
for i in range(np.shape(item_id)[1]):
x_tmp.append(self.out)
x = tf.stack(x_tmp)
print(np.shape(x))
print(np.shape(w_items))
x = tf.transpose(x, [1, 0, 2])
self.user_latent_factor = tf.nn.embedding_lookup(self.P, user_id)
u_tmp = []
for i in range(np.shape(item_id)[1]):
u_tmp.append(self.user_latent_factor)
u = tf.stack(u_tmp)
print(np.shape(u))
u = tf.transpose(u, [1, 0, 2])
self.user_specific_bias = tf.nn.embedding_lookup(self.V, user_id)
u_tmp_2 = []
for i in range(np.shape(item_id)[1]):
u_tmp_2.append(self.user_specific_bias)
u_2 = tf.stack(u_tmp_2)
print(np.shape(u_2))
u_2 = tf.transpose(u_2, [1, 0, 2])
self.alpha = 0.2
if isTrain:
res = self.alpha * tf.reduce_sum(
tf.nn.dropout(tf.square(w_items - u), 1),
2) + (1 - self.alpha) * tf.reduce_sum(
tf.nn.dropout(tf.square(x - w_items_2), 1),
2) #+ 0.1 * tf.reduce_sum(tf.square(x - u), 2)
else:
res = self.alpha * tf.reduce_sum(tf.square(w_items - u), 2) + (
1 - self.alpha) * tf.reduce_sum(tf.square(x - w_items_2), 2)
print(np.shape(res))
return tf.squeeze(res)
img = sample.reshape(32, 32,3)
plt.imshow(toimage(img),interpolation='nearest')
return fig
initializer = tf.contrib.layers.xavier_initializer()
rand_uniform = tf.random_uniform_initializer(-1,1,seed=2)
X = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
N = tf.placeholder(tf.float32, shape=[None, 100])
(x_train, y_train), (x_test, y_test) = load_data()
#x_train = np.concatenate((x_train, x_test), axis=0)
#y_train = np.concatenate((y_train, y_test), axis=0)
x_train = normalize(x_train)
y_train_one_hot = tf.squeeze(tf.one_hot(y_train, 10),axis=1)
theta_G =[]
def xavier_init(size):
in_dim = size[0]
xavier_stddev = 1. / tf.sqrt(in_dim / 2.)
return tf.random_normal(shape=size, stddev=xavier_stddev)
def autoencoder(x):
input_shape=[None, 32, 32, 3]
n_filters=[3, 128, 256, 512]
filter_sizes=[5, 5, 5, 5]
if len(x.get_shape()) == 3:
x_dim = np.sqrt(x.get_shape().as_list()[1])
def triangulate(startpoints, endpoints, weights, name="ray_triangulate"):
"""Triangulates 3d points by miminizing the sum of squared distances to rays.
The rays are defined by their start points and endpoints. At least two rays
are required to triangulate any given point. Contrary to the standard
reprojection-error metric, the sum of squared distances to rays can be
minimized in a closed form.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
startpoints: A tensor of ray start points with shape `[A1, ..., An, V, 3]`,
the number of rays V around which the solution points live should be
greater or equal to 2, otherwise triangulation is impossible.
endpoints: A tensor of ray endpoints with shape `[A1, ..., An, V, 3]`, the
number of rays V around which the solution points live should be greater
or equal to 2, otherwise triangulation is impossible. The `endpoints`
tensor should have the same shape as the `startpoints` tensor.
weights: A tensor of ray weights (certainties) with shape `[A1, ..., An,
V]`. Weights should have all positive entries. Weight should have at least
two non-zero entries for each point (at least two rays should have
certainties > 0).
name: A name for this op. The default value of None means "ray_triangulate".
Returns:
A tensor of triangulated points with shape `[A1, ..., An, 3]`.
Raises:
ValueError: If the shape of the arguments is not supported.
"""
with tf.name_scope(name):
startpoints = tf.convert_to_tensor(value=startpoints)
endpoints = tf.convert_to_tensor(value=endpoints)
weights = tf.convert_to_tensor(value=weights)
shape.check_static(
tensor=startpoints,
tensor_name="startpoints",
has_rank_greater_than=1,
has_dim_equals=(-1, 3),
has_dim_greater_than=(-2, 1))
shape.check_static(
tensor=endpoints,
tensor_name="endpoints",
has_rank_greater_than=1,
has_dim_equals=(-1, 3),
has_dim_greater_than=(-2, 1))
shape.compare_batch_dimensions(
tensors=(startpoints, endpoints, weights),
last_axes=(-2, -2, -1),
broadcast_compatible=False)
weights = asserts.assert_all_above(weights, 0.0, open_bound=False)
weights = asserts.assert_at_least_k_non_zero_entries(weights, k=2)
left_hand_side_list = []
right_hand_side_list = []
# TODO(b/130892100): Replace the inefficient for loop and add comments here.
for ray_id in range(weights.shape[-1]):
weights_single_ray = weights[..., ray_id]
startpoints_single_ray = startpoints[..., ray_id, :]
endpoints_singleview = endpoints[..., ray_id, :]
ray = endpoints_singleview - startpoints_single_ray
ray = tf.nn.l2_normalize(ray, axis=-1)
ray_x, ray_y, ray_z = tf.unstack(ray, axis=-1)
zeros = tf.zeros_like(ray_x)
cross_product_matrix = tf.stack(
(zeros, -ray_z, ray_y, ray_z, zeros, -ray_x, -ray_y, ray_x, zeros),
axis=-1)
cross_product_matrix_shape = tf.concat(
(tf.shape(input=cross_product_matrix)[:-1], (3, 3)), axis=-1)
cross_product_matrix = tf.reshape(
cross_product_matrix, shape=cross_product_matrix_shape)
weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
left_hand_side = weights_single_ray * cross_product_matrix
left_hand_side_list.append(left_hand_side)
dot_product = tf.matmul(cross_product_matrix,
tf.expand_dims(startpoints_single_ray, axis=-1))
right_hand_side = weights_single_ray * dot_product
right_hand_side_list.append(right_hand_side)
left_hand_side_multi_rays = tf.concat(left_hand_side_list, axis=-2)
right_hand_side_multi_rays = tf.concat(right_hand_side_list, axis=-2)
points = tf.linalg.lstsq(left_hand_side_multi_rays,
right_hand_side_multi_rays)
points = tf.squeeze(points, axis=-1)
return points
def train(self):
Weight = tf.Variable(
tf.truncated_normal([self.corpus.n_words, const.EMBEDDING_SIZE],
stddev=1.0 / math.sqrt(const.EMBEDDING_SIZE)))
bias = tf.Variable(tf.random_normal([self.corpus.n_words]))
inputs = tf.placeholder(tf.int32, [const.BATCH_SIZE, 1])
targets = tf.placeholder(tf.int32, [const.BATCH_SIZE, 1])
vocabs = tf.placeholder(tf.int32,
[const.BATCH_SIZE, self.corpus.n_words])
embed_weight_v = tf.Variable(
tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
1.0))
embed_weight_u = tf.Variable(
tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
1.0))
embed_weight_actual = tf.Variable(
tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
1.0))
embed_v = tf.nn.embedding_lookup(embed_weight_v, inputs)
embed_u = tf.nn.embedding_lookup(embed_weight_u, targets)
embed_actual = tf.nn.embedding_lookup(embed_weight_actual, vocabs)
'''
print(embed_u.shape)
print(embed_v.shape)
print(embed_actual.shape)
exit()
'''
embed_v_trans = tf.transpose(embed_v, [0, 2, 1])
#print(embed_v_trans.shape)
scores = tf.squeeze(tf.matmul(embed_u, embed_v_trans),
[2]) # batch_size x 1
norm_scores = tf.squeeze(tf.matmul(embed_actual, embed_v_trans),
[2]) # batch_size x input_size
softmax = tf.exp(scores) / tf.reduce_sum(tf.exp(norm_scores), 1)
softmax = tf.expand_dims(softmax, 1)
nll_loss = -tf.reduce_mean(
tf.log(tf.clip_by_value(softmax, 1e-10, 1.0)))
optimizer = tf.train.AdamOptimizer(
learning_rate=const.LR_RATE).minimize(nll_loss)
saver = tf.train.Saver()
losses = []
with tf.Session() as sess:
tf.global_variables_initializer().run()
for epoch in range(const.EPOCH):
for i, batch in enumerate(
self.corpus.batch_data(const.BATCH_SIZE)):
_inputs, _targets = zip(*batch) # unzip
_inputs = np.hstack(_inputs) # (2, )
_inputs = _inputs.reshape(_inputs.shape[0], 1)
_targets = np.vstack(_targets) # (2, 1)
vocab = self.corpus.var_sentence(self.corpus.vocab)
_vocabs = []
[_vocabs.append(vocab) for x in range(inputs.shape[0])]
_vocabs = np.array(_vocabs)
_, _loss = sess.run([optimizer, nll_loss],
feed_dict={
inputs: _inputs,
targets: _targets,
vocabs: _vocabs
})
losses.append(_loss)
if i % 500:
print('i, ', i, 'loss', _loss)
if epoch % 10 == 0:
print('epoch, ', epoch, 'mean loss', np.mean(losses))
losses = []
# save model
saver.save(sess, const.MODEL_PATH)
def sub_func(example_proto):
features = {}
label = {}
for col in used_fea.categorical_columns:
if col not in ['appId_list_encoded']:
features[col] = tf.FixedLenFeature((1),
tf.int64,
default_value=0)
for col in used_fea.numerical_columns:
features[col] = tf.FixedLenFeature((1),
tf.float32,
default_value=0.1)
features['appId_list_encoded'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_appId_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_duration_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_times_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_use_date_list'] = tf.VarLenFeature(dtype=tf.int64)
features['all_activedApp_cate_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_appId_duration_list'] = tf.VarLenFeature(
dtype=tf.int64)
features['usage_appId_times_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_appId_mean_dura_list'] = tf.VarLenFeature(
dtype=tf.int64)
features['usage_appId_full_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_duration_full_list'] = tf.VarLenFeature(dtype=tf.int64)
features['usage_time_full_list'] = tf.VarLenFeature(dtype=tf.int64)
parsed_features = tf.parse_single_example(example_proto, features)
parsed_features['appId_list_encoded'] = tf.sparse_tensor_to_dense(
parsed_features['appId_list_encoded'])
parsed_features['usage_appId_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_appId_list'])
parsed_features['usage_duration_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_duration_list'])
parsed_features['usage_times_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_times_list'])
parsed_features['usage_use_date_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_use_date_list'])
parsed_features[
'all_activedApp_cate_list'] = tf.sparse_tensor_to_dense(
parsed_features['all_activedApp_cate_list'])
parsed_features[
'usage_appId_duration_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_appId_duration_list'])
parsed_features['usage_appId_times_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_appId_times_list'])
parsed_features[
'usage_appId_mean_dura_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_appId_mean_dura_list'])
parsed_features['usage_appId_full_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_appId_full_list'])
parsed_features[
'usage_duration_full_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_duration_full_list'])
parsed_features['usage_time_full_list'] = tf.sparse_tensor_to_dense(
parsed_features['usage_time_full_list'])
label['age_group'] = tf.FixedLenFeature((1),
tf.float32,
default_value=1)
parsed_label = tf.parse_single_example(example_proto, label)
parsed_label['age_group'] = tf.cast(parsed_label['age_group'],
tf.int64)
label = tf.cast(tf.one_hot(parsed_label['age_group'] - 1, 6, 1, 0),
tf.float32)
label = tf.squeeze(label, squeeze_dims=[0])
return parsed_features, label
def sample(self,
ztype='prior',
num_samples=1,
seed=None,
linear_predictors=None,
checkpoint_file=None,
z_in=None,
y_in=None,
gmm_params=None,
mark_probs=None):
"""
Generate samples from prior/posterior and model
Args:
ztype (str): distribution used for latent state samples
'prior' | 'posterior'
num_samples (int, optional)
seed (int, optional): random seed for reproducibly generating
random samples
linear_predictors (list of np arrays):
1 x num_time_pts x dim_pred;
there will be num_samples random samples for this one example
of each linear predictor
checkpoint_file (str, optional): checkpoint file specifying model
from which to generate samples; if `None`, will then look for a
checkpoint file created upon model initialization
Returns:
num_samples x num_time_pts x dim_obs numpy array: y
num_samples x num_time_pts x dim_latent numpy array: z
Raises:
ValueError: for incorrect `ztype` values
"""
self._check_graph()
# intialize session
with tf.Session(graph=self.graph, config=self.sess_config) as sess:
self.restore_model(sess, checkpoint_file=checkpoint_file)
# print(self.gen_net.networks[0].layers[0].kernel.eval(sess))
if z_in is not None:
# this only works in one specific case
ztmp = tf.convert_to_tensor(z_in, dtype=self.dtype)
ymean = self.gen_net.networks[0].apply_network(ztmp)
if mark_probs is not None:
# if mark_probs is None:
# raise ValueError('Must provide mark probabilities to evaluate this model')
F = tf.expand_dims(ymean, axis=2)
W = tf.convert_to_tensor(mark_probs, dtype=self.dtype)
ymean = tf.squeeze(tf.matmul(F, W))
y_ = tf.random_poisson(lam=ymean,
shape=[1],
dtype=tf.float32,
seed=seed)
y = np.squeeze(y_.eval())
z = z_in
# [y, z] = sess.run([y_, ztmp], feed_dict=feed_dict)
elif ztype is 'prior':
y, z = self.gen_net.sample(sess, num_samples, seed,
linear_predictors, mark_probs)
elif ztype is 'posterior':
if y_in is None:
raise ValueError(
'`y_in` required to sample from posterior')
# TODO: needs input to inference network as well
z = self.inf_net.sample(sess,
y_in,
mark_probs=mark_probs,
seed=seed)
ztmp = tf.convert_to_tensor(z, dtype=self.dtype)
ymean = self.gen_net.networks[0].apply_network(ztmp)
if mark_probs is not None:
# if mark_probs is None:
# raise ValueError('Must provide mark probabilities to evaluate this model')
F = tf.transpose(ymean, (0, 2, 1, 3))
W = tf.convert_to_tensor(mark_probs, dtype=self.dtype)
ymean = tf.transpose(tf.matmul(F, W), (0, 2, 1, 3))
y_ = tf.random_poisson(lam=ymean,
shape=[1],
dtype=tf.float32,
seed=seed)
y = np.squeeze(y_.eval())
else:
raise ValueError('Invalid string "%s" for ztype argument')
# simulate the marks -- takes a long time
if gmm_params is not None:
ztmp = tf.convert_to_tensor(z, dtype=self.dtype)
f_k = self.gen_net.networks[0].apply_network(ztmp)
f_k = f_k.eval()
f_k = np.reshape(f_k, (-1, f_k.shape[2]))
y_flat = np.reshape(y, (-1, self.dim_obs[0]))
mrk = []
ii = 0
for i in range(self.dim_obs[0]):
mus = gmm_params[i][0]
sigs = gmm_params[i][1]
pi = gmm_params[i][2]
dim_marks = mus.shape[0]
k = pi.shape[0]
f_ik = f_k[:, ii:(ii + k)]
ii += k
log_pi_hat = np.log(f_ik) + np.log(pi +
1e-16)[np.newaxis, :]
c = log_pi_hat.max(1)[:, np.newaxis]
log_norm = c + np.log(
np.sum(np.exp(log_pi_hat - c), axis=1))[:, np.newaxis]
log_pi_hat -= log_norm
pi_hat = np.exp(log_pi_hat)
temp_mrk = np.zeros((0, dim_marks))
ind = np.zeros(0)
for t in np.where(y_flat[:, i] > 0)[0]:
Nt = int(y_flat[t, i])
zeta = rnd.multinomial(1, pi_hat[t, :],
size=Nt).argmax(1)
mu = mus[:, zeta]
C = sigs[:, :, zeta]
m_t = np.array([rnd.multivariate_normal(mu[:,jj],C[:,:,jj]) \
for jj in range(Nt)], dtype = np.float32)
temp_mrk = np.append(temp_mrk, m_t, axis=0)
ind = np.append(ind, np.ones(Nt) * int(t))
mrk.append([temp_mrk, ind])
else:
mrk = None
return y, z, mrk
def computemodel(self, is_resnet=False, is_forcepos=False):
''' Perform Multiple Scattering here
1.) Multiple Scattering is perfomed by slice-wise propagation the E-Field throught the sample
2.) Each Field has to be backprojected to the BFP
3.) Last step is to ceate a focus-stack and sum over all angles
This is done for all illumination angles (coming from illumination NA
simultaneasly)'''
self.is_resnet = is_resnet
self.is_forcepos = is_forcepos
print("Buildup Q-PHASE Model ")
###### make sure, that the first dimension is "batch"-size; in this case it is the illumination number
# @beniroquai It's common to have to batch dimensions first and not last.!!!!!
# the following loop propagates the field sequentially to all different Z-planes
## propagate the field through the entire object for all angles simultaneously
A_prop = np.transpose(
self.A_input,
[3, 0, 1, 2
]) # ??????? what the hack is happening with transpose?!
if (self.is_tomo):
print('Experimentally using the tomographic scheme!')
A_prop = np.conj(A_prop)
myprop = np.exp(1j * self.dphi) * (self.dphi > 0)
# excludes the near field components in each step
myprop = tf_helper.repmat4d(np.fft.fftshift(myprop), self.Nc)
myprop = np.transpose(
myprop, [3, 0, 1, 2
]) # ??????? what the hack is happening with transpose?!
print('--------> ATTENTION: I added a pi factor - is this correct?!')
RefrEffect = np.squeeze(1j * np.pi * self.dz * self.k0 * self.RefrCos)
# Precalculate the oblique effect on OPD to speed it up
# for now orientate the dimensions as (alpha_illu, x, y, z) - because tensorflow takes the first dimension as batch size
with tf.name_scope('Variable_assignment'):
self.TF_A_input = tf.constant(A_prop, dtype=tf.complex64)
self.TF_RefrEffect = tf.reshape(
tf.constant(RefrEffect, dtype=tf.complex64), [self.Nc, 1, 1])
self.TF_myprop = tf.constant(np.squeeze(myprop),
dtype=tf.complex64)
self.TF_Po = tf.cast(tf.constant(self.Po), tf.complex64)
self.TF_Zernikes = tf.constant(self.myzernikes, dtype=tf.float32)
self.TF_myAllSlicePropagator = tf.constant(
self.myAllSlicePropagator, dtype=tf.complex64)
# Only update those Factors which are really necesarry (e.g. Defocus is not very likely!)
self.TF_zernikefactors = tf.Variable(self.zernikefactors,
dtype=tf.float32,
name='var_zernikes')
#indexes = tf.constant([[4], [5], [6], [7], [8], [9]])
indexes = tf.cast(tf.where(tf.constant(self.zernikemask) > 0),
tf.int32)
updates = tf.gather_nd(self.TF_zernikefactors, indexes)
# Take slice
# Build tensor with "filtered" gradient
part_X = tf.scatter_nd(indexes, updates,
tf.shape(self.TF_zernikefactors))
self.TF_zernikefactors_filtered = part_X + tf.stop_gradient(
-part_X + self.TF_zernikefactors)
# TODO: Introduce the averraged RI along Z - MWeigert
self.TF_A_prop = tf.squeeze(self.TF_A_input)
self.U_z_list = []
# Initiliaze memory
self.allSumAmp = 0
self.TF_allSumAmp = tf.zeros([self.mysize[0], self.Nx, self.Ny],
dtype=tf.complex64)
''' Eventually add a RESNET-layer between RI and Microscope to correct for model discrepancy?'''
if (self.is_resnet):
with tf.variable_scope('res_real', reuse=False):
TF_real_3D = self.residual_block(
tf.expand_dims(tf.expand_dims(self.TF_obj, 3), 0), 1, True)
TF_real_3D = tf.squeeze(TF_real_3D)
with tf.variable_scope('res_imag', reuse=False):
TF_imag_3D = self.residual_block(
tf.expand_dims(tf.expand_dims(self.TF_obj_absorption, 3),
0), 1, True)
TF_imag_3D = tf.squeeze(TF_imag_3D)
else:
TF_real_3D = self.TF_obj
TF_imag_3D = self.TF_obj_absorption
# wrapper for force-positivity on the RI-instead of penalizing it
if (self.is_forcepos):
print('----> ATTENTION: We add the PreMonotonicPos')
TF_real_3D = tf_reg.PreMonotonicPos(TF_real_3D)
TF_imag_3D = tf_reg.PreMonotonicPos(TF_imag_3D)
TF_A_prop_illu_list = []
# simulate multiple scattering through object
with tf.name_scope('Fwd_Propagate'):
#print('---------ATTENTION: We are inverting the RI!')
# First Iterate over all illumination angles
for pillu in range(0, self.Nc):
# Second iterate over all z-slices
TF_A_prop_z_tmp = self.TF_A_prop[pillu, :, :]
for pz in range(0, self.mysize[0]):
if (self.is_padding):
tf_paddings = tf.constant([[
self.mysize_old[1] // 2, self.mysize_old[1] // 2
], [self.mysize_old[2] // 2, self.mysize_old[2] // 2]])
TF_real = tf.pad(TF_real_3D[-pz, :, :],
tf_paddings,
mode='CONSTANT',
name='TF_obj_real_pad')
TF_imag = tf.pad(TF_imag_3D[-pz, :, :],
tf_paddings,
mode='CONSTANT',
name='TF_obj_imag_pad')
else:
TF_real = (TF_real_3D[-pz, :, :])
TF_imag = (TF_imag_3D[-pz, :, :])
self.TF_f = tf.exp(self.TF_RefrEffect[pillu, :, :] *
tf.complex(TF_real, TF_imag))
#self.TF_A_prop = tf.Print(self.TF_A_prop, [self.tf_iterator], 'Prpagation step: ')
with tf.name_scope('Refract'):
# beware the "i" is in TF_RefrEffect already!
TF_A_prop_z_tmp = TF_A_prop_z_tmp * self.TF_f # refraction step
with tf.name_scope('Propagate'):
TF_A_prop_z_tmp = tf.ifft2d(
tf.fft2d(TF_A_prop_z_tmp) *
self.TF_myprop[pillu, :, :]) # diffraction step
TF_A_prop_illu_list.append(TF_A_prop_z_tmp)
self.TF_A_prop = tf.stack(TF_A_prop_illu_list)
# in a final step limit this to the detection NA:
self.TF_Po_aberr = tf.exp(1j * tf.cast(
tf.reduce_sum(self.TF_zernikefactors_filtered * self.TF_Zernikes,
axis=2), tf.complex64)) * self.TF_Po
self.TF_A_prop = tf.ifft2d(
tf.fft2d(self.TF_A_prop) * self.TF_Po * self.TF_Po_aberr)
# Experimenting with pseudo tomographic data?
if self.is_tomo:
print('Only Experimental! Tomographic data?!')
# Bring the slice back to focus - does this make any sense?!
print('----------> Bringing field back to focus')
TF_centerprop = tf.exp(
-1j *
tf.cast(self.Nz / 2 * tf.angle(self.TF_myprop), tf.complex64))
self.TF_A_prop = tf.ifft2d(
tf.fft2d(self.TF_A_prop) * TF_centerprop) # diffraction step
return self.TF_A_prop
# create Z-Stack by backpropagating Information in BFP to Z-Position
self.kzcoord = np.reshape(self.kz[self.Ic > 0], [1, 1, 1, self.Nc])
# self.mid3D = ([np.int(np.ceil(self.A_input.shape[0] / 2) + 1), np.int(np.ceil(self.A_input.shape[1] / 2) + 1), np.int(np.ceil(self.mysize[0] / 2) + 1)])
self.mid3D = ([
np.int(self.mysize[0] // 2),
np.int(self.A_input.shape[0] // 2),
np.int(self.A_input.shape[1] // 2)
])
with tf.name_scope('Back_Propagate'):
print('----------> ATTENTION: PHASE SCRAMBLING!')
for pillu in range(0, self.Nc):
TF_allAmp_list = []
# fprintf('BackpropaAngle no: #d\n',pillu);
OneAmp = tf.expand_dims(self.TF_A_prop[pillu, :, :], 0)
# Fancy backpropagation assuming what would be measured if the sample was moved under oblique illumination:
# The trick is: First use conceptually the normal way
# and then apply the XYZ shift using the Fourier shift theorem (corresponds to physically shifting the object volume, scattered field stays the same):
self.TF_AdjustKXY = tf.squeeze(
tf.conj(self.TF_A_input[pillu, :, :, ])
) # Maybe a bit of a dirty hack, but we first need to shift the zero coordinate to the center
self.TF_AdjustKZ = tf.cast(
tf.transpose(
np.exp(2 * np.pi * 1j * self.dz * np.reshape(
np.arange(
0, self.mysize[0], 1
), # We want to start from first Z-slice then go to last which faces the objective lens
[1, 1, self.mysize[0]]) *
self.kzcoord[:, :, :, pillu]),
[2, 1, 0]),
tf.complex64)
for pz in range(0, self.Nz):
with tf.name_scope('Back_Propagate_Step'):
TF_allAmp_list.append(
tf.squeeze(
tf.ifft2d(
tf.fft2d(OneAmp) *
self.TF_myAllSlicePropagator[pz, :, :])) *
self.TF_AdjustKZ[pz, :, :] *
self.TF_AdjustKXY[:, :]
) # * (TF_AdjustKZ); # 2x bfxfun. Propagates a single amplitude pattern back to the whole stack
#tf_global_phase = tf.cast(tf.angle(self.TF_allAmp[self.mid3D[0],self.mid3D[1],self.mid3D[2]]), tf.complex64)
#tf_global_phase = tf.cast(np.random.randn(1)*np.pi,tf.complex64)
#self.TF_allAmp = self.TF_allAmp * tf.exp(1j*tf_global_phase) # Global Phases need to be adjusted at this step! Use the zero frequency
TF_allAmp = tf.stack(TF_allAmp_list)
with tf.name_scope('Adjust_Global_Phase'):
self.TF_allAmp_3dft = tf.fft3d(
tf.expand_dims(TF_allAmp, axis=0))
tf_global_phase = tf.angle(
self.TF_allAmp_3dft[0, 0, 0, 0]
) #tf.angle(self.TF_allAmp_3dft[0, self.mid3D[2], self.mid3D[1], self.mid3D[0]])
tf_global_phase = tf.cast(tf_global_phase, tf.complex64)
TF_allAmp = TF_allAmp * tf.exp(-1j * tf_global_phase)
# Global Phases need to be adjusted at this step! Use the zero frequency
#print('Global phase: '+str(tf.exp(1j*tf.cast(tf.angle(self.TF_allAmp[self.mid3D[0],self.mid3D[1],self.mid3D[2]]), tf.complex64).eval()))
with tf.name_scope(
'Sum_Amps'
): # Normalize amplitude by condenser intensity
self.TF_allSumAmp += TF_allAmp
# Superpose the Amplitudes
# print('Current illumination angle # is: ' + str(pillu))
# Normalize the image such that the values do not depend on the fineness of
# the source grid.
self.TF_allSumAmp = self.TF_allSumAmp / self.Nc #/tf.cast(tf.reduce_max(tf.abs(self.TF_allSumAmp)), tf.complex64)
# Following is the normalization according to Martin's book. It ensures
# that a transparent specimen is imaged with unit intensity.
# normfactor=abs(Po).^2.*abs(Ic); We do not use it, because it leads to
# divide by zero for dark-field system. Instead, through normalizations
# perfomed above, we ensure that image of a point under matched
# illumination is unity. The brightness of all the other configurations is
# relative to this benchmark.
#
# negate padding
if self.is_padding:
self.TF_allSumAmp = self.TF_allSumAmp[:, self.Nx // 2 -
self.Nx // 4:self.Nx // 2 +
self.Nx // 4, self.Ny // 2 -
self.Ny // 4:self.Ny // 2 +
self.Ny // 4]
return self.TF_allSumAmp
def inference(batch_placeholders, similarity_placeholder, init_word_embeds,
word_to_num, num_to_word):
print("Begin inference:")
print("Creating variables")
E = tf.Variable(init_word_embeds, dtype=tf.float32)
W = tf.Variable(
tf.random_uniform(
[params.lstm_size, params.lstm_size, params.slice_size],
minval=-1.0 / params.lstm_size,
maxval=1.0 / params.lstm_size,
name='W'))
V = tf.Variable(
tf.random_uniform([params.slice_size, 2 * params.lstm_size],
minval=-1.0 / (2 * params.lstm_size),
maxval=1.0 / (2 * params.lstm_size)))
b = tf.Variable(tf.zeros([1, params.slice_size]), name='b')
U = tf.Variable(
tf.random_uniform([1, params.slice_size],
minval=-1.0 / params.slice_size,
maxval=1.0 / params.slice_size))
lstm = createLSTM(params.lstm_size)
print("Calcing sentences2vec")
question_vec, pos_answer_vec, neg1, neg2, neg3 = tf.split(
1, params.corrupt_size + 2, batch_placeholders)
#scr_pos_answer, scr_neg1 , scr_neg2 , scr_neg3 = tf.split(1, params.corrupt_size+1,similarity_placeholder)
#similarity_scores = tf.cast(similarity_placeholder, tf.float32)
question_vec = tf.squeeze(question_vec)
pos_answer_vec = tf.squeeze(pos_answer_vec)
neg1 = tf.squeeze(neg1)
neg2 = tf.squeeze(neg2)
neg3 = tf.squeeze(neg3)
#scr_pos_answer = tf.squeeze(scr_pos_answer)
#scr_neg1 = tf.squeeze(scr_neg1)
#scr_neg2 = tf.squeeze(scr_neg2)
#scr_neg3 = tf.squeeze(scr_neg3)
#question_vec = tf.reduce_mean(tf.gather(E,question_vec),1)
question_vec = train_sentence2vectorLSTM(lstm, E, question_vec, False)
pos_answer_vec = train_sentence2vectorLSTM(lstm, E, pos_answer_vec, True)
neg1 = train_sentence2vectorLSTM(lstm, E, neg1, True)
neg2 = train_sentence2vectorLSTM(lstm, E, neg2, True)
neg3 = train_sentence2vectorLSTM(lstm, E, neg3, True)
#new_p = tf.zeros([pparams.lstm_size+1])
#pos_answer_vec = tf.reshape(pos_answer_vec, [-1])
#print scr_pos_answer.get_shape
#pos_answer_vec = tf.concat(1,[pos_answer_vec,scr_pos_answer])
#neg1 = tf.concat(1,[neg1,scr_neg1])
#neg2 = tf.concat(1,[neg2,scr_neg2])
#neg3 = tf.concat(1,[neg3,scr_neg3])
#pos_answer_vec = tf.reduce_mean(tf.gather(E, pos_answer_vec), 1)
#neg1 = tf.reduce_mean(tf.gather(E, neg1), 1)
#neg2 = tf.reduce_mean(tf.gather(E, neg2), 1)
#neg3 = tf.reduce_mean(tf.gather(E, neg3), 1)
tensors = []
for i in range(params.slice_size):
tensor = tf.reduce_sum(
pos_answer_vec * tf.matmul(question_vec, W[:, :, i]), 1)
tensors.append(tensor)
score_pos = tf.pack(tensors)
vec_concat = tf.transpose(
tf.matmul(V, tf.transpose(tf.concat(1,
[question_vec, pos_answer_vec]))))
score_pos = tf.matmul(tf.nn.relu(tf.transpose(score_pos) + vec_concat + b),
tf.transpose(U))
negative = []
for i in [neg1, neg2, neg3]:
tensors = []
for j in range(params.slice_size):
tensor = tf.reduce_sum(i * tf.matmul(question_vec, W[:, :, j]), 1)
tensors.append(tensor)
score_neg = tf.pack(tensors)
vec_concat = tf.transpose(
tf.matmul(V, tf.transpose(tf.concat(1, [question_vec, i]))))
score_neg = tf.matmul(
tf.nn.relu(tf.transpose(score_neg) + vec_concat + b),
tf.transpose(U))
negative.append(score_neg)
return [score_pos, negative[0], negative[1], negative[2]]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='data/',
help='data directory containing input.txt')
parser.add_argument('--batch_size', type=int, default=120,
help='minibatch size')
parser.add_argument('--win_size', type=int, default=5,
help='context sequence length')
parser.add_argument('--hidden_num', type=int, default=100,
help='number of hidden layers')
parser.add_argument('--word_dim', type=int, default=300,
help='number of word embedding')
parser.add_argument('--num_epochs', type=int, default=3,
help='number of epochs')
parser.add_argument('--grad_clip', type=float, default=10.,
help='clip gradients at this value')
args = parser.parse_args()
args_msg = '\n'.join([ arg + ': ' + str(getattr(args, arg)) for arg in vars(args)])
data_loader = TextLoader(args.data_dir, args.batch_size, args.win_size)
args.vocab_size = data_loader.vocab_size
graph = tf.Graph()
with graph.as_default():
input_data = tf.placeholder(tf.int64, [args.batch_size, args.win_size])
targets = tf.placeholder(tf.int64, [args.batch_size, 1])
with tf.variable_scope('nnlm' + 'embedding'):
embeddings = tf.Variable(tf.random_uniform([args.vocab_size, args.word_dim], -1.0, 1.0))
embeddings = tf.nn.l2_normalize(embeddings, 1)
with tf.variable_scope('nnlm' + 'weight'):
weight_h = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.hidden_num],
stddev=1.0 / math.sqrt(args.hidden_num)))
softmax_w = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.vocab_size],
stddev=1.0 / math.sqrt(args.win_size * args.word_dim)))
softmax_u = tf.Variable(tf.truncated_normal([args.hidden_num, args.vocab_size],
stddev=1.0 / math.sqrt(args.hidden_num)))
b_1 = tf.Variable(tf.random_normal([args.hidden_num]))
b_2 = tf.Variable(tf.random_normal([args.vocab_size]))
def infer_output(input_data):
"""
hidden = tanh(x * H + b_1)
output = softmax(x * W + hidden * U + b_2)
"""
input_data_emb = tf.nn.embedding_lookup(embeddings, input_data)
input_data_emb = tf.reshape(input_data_emb, [-1, args.win_size * args.word_dim])
hidden = tf.tanh(tf.matmul(input_data_emb, weight_h)) + b_1
hidden_output = tf.matmul(hidden, softmax_u) + tf.matmul(input_data_emb, softmax_w) + b_2
output = tf.nn.softmax(hidden_output)
return output
outputs = infer_output(input_data)
one_hot_targets = tf.one_hot(tf.squeeze(targets), args.vocab_size, 1.0, 0.0)
loss = -tf.reduce_mean(tf.reduce_sum(tf.log(outputs) * one_hot_targets, 1))
# Clip grad.
optimizer = tf.train.AdagradOptimizer(0.1)
gvs = optimizer.compute_gradients(loss)
capped_gvs = [(tf.clip_by_value(grad, -args.grad_clip, args.grad_clip), var) for grad, var in gvs]
optimizer = optimizer.apply_gradients(capped_gvs)
embeddings_norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / embeddings_norm
processing_message_lst = list()
with tf.Session(graph=graph) as sess:
tf.global_variables_initializer().run()
for e in range(args.num_epochs):
data_loader.reset_batch_pointer()
for b in range(data_loader.num_batches):
start = time.time()
x, y = data_loader.next_batch()
feed = {input_data: x, targets: y}
train_loss, _ = sess.run([loss, optimizer], feed)
end = time.time()
processing_message = "{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
b, data_loader.num_batches,
e, train_loss, end - start)
print(processing_message)
processing_message_lst.append(processing_message)
# print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
# b, data_loader.num_batches,
# e, train_loss, end - start))
np.save('nnlm_word_embeddings.zh', normalized_embeddings.eval())
# record training processing
print(start - end)
local_time = str(time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()))
with open("{}.txt".format('casdsa'), 'w', encoding='utf-8') as f:
f.write(local_time)
f.write(args_msg)
f.write('\n'.join(processing_message_lst))
def body(self, features):
hparams = self.hparams
batch_size = common_layers.shape_list(features["inputs"])[0]
# Swap time and batch axes.
input_frames = common_video.swap_time_and_batch_axes(features["inputs"])
target_frames = common_video.swap_time_and_batch_axes(features["targets"])
# Get actions if exist otherwise use zeros
input_actions = self.get_input_if_exists(
features, "input_action", batch_size, hparams.video_num_input_frames)
target_actions = self.get_input_if_exists(
features, "target_action", batch_size, hparams.video_num_target_frames)
# Get rewards if exist otherwise use zeros
input_rewards = self.get_input_if_exists(
features, "input_reward", batch_size, hparams.video_num_input_frames)
target_rewards = self.get_input_if_exists(
features, "target_reward", batch_size, hparams.video_num_target_frames)
all_actions = tf.concat([input_actions, target_actions], axis=0)
all_rewards = tf.concat([input_rewards, target_rewards], axis=0)
all_frames = tf.concat([input_frames, target_frames], axis=0)
# Each image is being used twice, in latent tower and main tower.
# This is to make sure we are using the *same* image for both, ...
# ... given how TF queues work.
# NOT sure if this is required at all. Doesn"t hurt though! :)
all_frames = tf.identity(all_frames)
gen_images, gen_rewards, latent_means, latent_stds = self.construct_model(
images=all_frames,
actions=all_actions,
rewards=all_rewards,
)
extra_loss = self.get_extra_loss(
latent_means=latent_means,
latent_stds=latent_stds,
true_frames=all_frames,
gen_frames=gen_images)
# Visualize predictions in Tensorboard
if self.is_training:
self.visualize_predictions(all_frames[1:], gen_images)
# Ignore the predictions from the input frames.
# This is NOT the same as original paper/implementation.
predictions = gen_images[hparams.video_num_input_frames-1:]
reward_pred = gen_rewards[hparams.video_num_input_frames-1:]
reward_pred = tf.squeeze(reward_pred, axis=2) # Remove extra dimension.
# Swap back time and batch axes.
predictions = common_video.swap_time_and_batch_axes(predictions)
reward_pred = common_video.swap_time_and_batch_axes(reward_pred)
if self.is_training and hparams.internal_loss:
# add the loss for input frames as well.
extra_gts = all_frames[1:hparams.video_num_input_frames]
extra_gts = common_video.swap_time_and_batch_axes(extra_gts)
extra_pds = gen_images[:hparams.video_num_input_frames-1]
extra_pds = common_video.swap_time_and_batch_axes(extra_pds)
extra_raw_gts = features["inputs_raw"][:, 1:]
recon_loss = self.get_extra_internal_loss(
extra_raw_gts, extra_gts, extra_pds)
extra_loss += recon_loss
return_targets = predictions
if hparams.reward_prediction:
return_targets = {"targets": predictions, "target_reward": reward_pred}
return return_targets, extra_loss
def get_model(point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output BxNx50 """
batch_size = point_cloud.get_shape()[0].value # batch_size
num_point = point_cloud.get_shape()[1].value # num_point
end_points = {}
with tf.variable_scope('transform_net1') as sc:
transform = input_transform_net(point_cloud,
is_training,
bn_decay,
K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = tf_util.conv2d(input_image,
64, [1, 3],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv1',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv2',
bn_decay=bn_decay)
with tf.variable_scope('transform_net2') as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
point_feat = tf.expand_dims(net_transformed, [2])
print(point_feat)
net = tf_util.conv2d(point_feat,
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv3',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv4',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
1024, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv5',
bn_decay=bn_decay)
global_feat = tf_util.max_pool2d(net, [num_point, 1],
padding='VALID',
scope='maxpool')
print(global_feat)
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
concat_feat = tf.concat(3, [point_feat, global_feat_expand])
print(concat_feat)
# 定义分割的mpl512-256-128 128-m, m为点所属的类别数目
net = tf_util.conv2d(concat_feat,
512, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv6',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
256, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv7',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv8',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv9',
bn_decay=bn_decay)
net = tf_util.conv2d(net,
50, [1, 1],
padding='VALID',
stride=[1, 1],
activation_fn=None,
scope='conv10')
net = tf.squeeze(net, [2]) # BxNxC
return net, end_points
def test_model_serving(self):
"""
Train a simple model and test serving flow by loading the SavedModel
"""
# Test model training on TFRecord SequenceExample data
data_dir = os.path.join(self.root_data_dir, "tfrecord")
feature_config: FeatureConfig = self.get_feature_config()
metrics_keys = ["categorical_accuracy"]
def get_dataset(parse_tfrecord):
return RelevanceDataset(
data_dir=data_dir,
data_format=DataFormatKey.TFRECORD,
feature_config=feature_config,
tfrecord_type=self.args.tfrecord_type,
max_sequence_size=self.args.max_sequence_size,
batch_size=self.args.batch_size,
preprocessing_keys_to_fns={},
train_pcent_split=self.args.train_pcent_split,
val_pcent_split=self.args.val_pcent_split,
test_pcent_split=self.args.test_pcent_split,
use_part_files=self.args.use_part_files,
parse_tfrecord=parse_tfrecord,
file_io=self.file_io,
logger=self.logger,
)
# Get raw TFRecord dataset
raw_dataset = get_dataset(parse_tfrecord=False)
# Parse the raw TFRecord dataset
parsed_dataset = get_dataset(parse_tfrecord=True)
model: RankingModel = self.get_ranking_model(
loss_key=self.args.loss_key, feature_config=feature_config, metrics_keys=metrics_keys
)
model.fit(dataset=parsed_dataset, num_epochs=1, models_dir=self.output_dir)
model.save(
models_dir=self.args.models_dir,
preprocessing_keys_to_fns={},
postprocessing_fn=None,
required_fields_only=not self.args.use_all_fields_at_inference,
pad_sequence=self.args.pad_sequence_at_inference,
)
# Load SavedModel and get the right serving signature
default_model = kmodels.load_model(
os.path.join(self.output_dir, "final", "default"), compile=False
)
assert ServingSignatureKey.DEFAULT in default_model.signatures
default_signature = default_model.signatures[ServingSignatureKey.DEFAULT]
tfrecord_model = kmodels.load_model(
os.path.join(self.output_dir, "final", "tfrecord"), compile=False
)
assert ServingSignatureKey.TFRECORD in tfrecord_model.signatures
tfrecord_signature = tfrecord_model.signatures[ServingSignatureKey.TFRECORD]
# Fetch a single batch for testing
sequence_example_protos = next(iter(raw_dataset.test))
parsed_sequence_examples = next(iter(parsed_dataset.test))[0]
parsed_dataset_batch = parsed_dataset.test.take(1)
# Use the loaded serving signatures for inference
model_predictions = model.predict(parsed_dataset_batch)[self.args.output_name].values
default_signature_predictions = default_signature(**parsed_sequence_examples)[
self.args.output_name
]
# Since we do not pad dummy records in tfrecord serving signature,
# we can only predict on a single record at a time
tfrecord_signature_predictions = [
tfrecord_signature(protos=tf.gather(sequence_example_protos, [i]))[
self.args.output_name
]
for i in range(self.args.batch_size)
]
# Get mask for padded values
mask = flatten_query(parsed_sequence_examples["mask"])
# Flatten scores to each record and filter out scores from padded records
default_signature_predictions = tf.squeeze(
filter_records(flatten_query(default_signature_predictions), mask), axis=-1
)
tfrecord_signature_predictions = tf.squeeze(
tf.concat(tfrecord_signature_predictions, axis=1)
)
# Compare the scores from the different versions of the model
assert np.isclose(model_predictions, default_signature_predictions, rtol=0.01,).all()
assert np.isclose(model_predictions, tfrecord_signature_predictions, rtol=0.01,).all()
assert np.isclose(
default_signature_predictions, tfrecord_signature_predictions, rtol=0.01,
).all()
def VINet_final(opt,
masked_img,
mask,
prev_state=None,
prev_feed=None,
idx=0,
trainable=True,
namescope='module'):
with tf.variable_scope(namescope):
# masked_img: b x C x TxHxW
T = masked_img.get_shape().as_list()[2]
ref_idx = (T - 1) // 2
ones = tf.ones_like(mask)
# encoder
enc_output = []
enc_input = tf.concat([masked_img, ones, ones * mask], axis=1)
# print(type(enc_input[:,:,ref_idx,:,:]), type(mask))
# input('hereeeee')
f1, f1_64, f1_128, f1_256 = VI_2D_Encoder_3(enc_input[:, :,
ref_idx, :, :],
opt,
reuse=False,
trainable=trainable,
namescope='encoder1')
f2, f2_64, f2_128, _ = VI_2D_Encoder_3(enc_input[:, :,
ref_idx - 2, :, :],
opt,
reuse=False,
trainable=trainable,
namescope='encoder2')
f3, f3_64, f3_128, _ = VI_2D_Encoder_3(enc_input[:, :,
ref_idx - 1, :, :],
opt,
reuse=True,
trainable=trainable,
namescope='encoder2')
f4, f4_64, f4_128, _ = VI_2D_Encoder_3(enc_input[:, :,
ref_idx + 1, :, :],
opt,
reuse=True,
trainable=trainable,
namescope='encoder2')
f5, f5_64, f5_128, _ = VI_2D_Encoder_3(enc_input[:, :,
ref_idx + 2, :, :],
opt,
reuse=True,
trainable=trainable,
namescope='encoder2')
f6, f6_64, f6_128, f6_256 = VI_2D_Encoder_3(prev_feed,
opt,
reuse=True,
trainable=trainable,
namescope='encoder2')
flow2 = LongFlowNetCorr(f1,
f2,
opt,
trainable=trainable,
reuse=False,
namescope='flownet')
flow3 = LongFlowNetCorr(f1,
f3,
opt,
trainable=trainable,
reuse=True,
namescope='flownet')
flow4 = LongFlowNetCorr(f1,
f4,
opt,
trainable=trainable,
reuse=True,
namescope='flownet')
flow5 = LongFlowNetCorr(f1,
f5,
opt,
trainable=trainable,
reuse=True,
namescope='flownet')
flow6 = LongFlowNetCorr(f1,
f6,
opt,
trainable=trainable,
reuse=True,
namescope='flownet')
f2_warp = WarpingLayer(f2, flow2)
f3_warp = WarpingLayer(f3, flow3)
f4_warp = WarpingLayer(f4, flow4)
f5_warp = WarpingLayer(f5, flow5)
f6_warp = WarpingLayer(f6, flow6)
f_stack_oth = tf.stack([f2_warp, f3_warp, f4_warp, f5_warp, f6_warp],
axis=2)
f_agg = tf.squeeze(VI_Aggregator(f_stack_oth,
depth=5,
trainable=trainable,
namescope='st_agg'),
axis=2)
occlusion_mask = MaskEstimator_(tf.abs(f1 - f_agg),
trainable=trainable,
namescope='masknet')
f_syn = (1 - occlusion_mask) * f1 + occlusion_mask * f_agg
bott_input = tf.concat([f1, f_syn], axis=1)
output = VI_2D_BottleNeck(bott_input,
trainable=trainable,
namescope='bottleneck')
# CONV LSTM
state = ConvLSTM(output,
prev_state,
trainable=trainable,
namescope='convlstm')
bot_h, bot_w = output.get_shape().as_list()[2:4]
# ============================ SCALE - 1/4 : 64 =============================
# align_corners=True
flow2_64 = resize_bilinear(flow2, (bot_h * 2, bot_w * 2),
align_corners=True) * 2
flow3_64 = resize_bilinear(flow3, (bot_h * 2, bot_w * 2),
align_corners=True) * 2
flow4_64 = resize_bilinear(flow4, (bot_h * 2, bot_w * 2),
align_corners=True) * 2
flow5_64 = resize_bilinear(flow5, (bot_h * 2, bot_w * 2),
align_corners=True) * 2
flow6_64 = resize_bilinear(flow6, (bot_h * 2, bot_w * 2),
align_corners=True) * 2
f2_64_warp = WarpingLayer(f2_64, flow2_64)
f3_64_warp = WarpingLayer(f3_64, flow3_64)
f4_64_warp = WarpingLayer(f4_64, flow4_64)
f5_64_warp = WarpingLayer(f5_64, flow5_64)
f6_64_warp = WarpingLayer(f6_64, flow6_64)
flow2_64 = LongFlowNetCorr(f1_64,
f2_64_warp,
opt,
flow2_64,
trainable=trainable,
reuse=False,
namescope='flownet_64') + flow2_64
flow3_64 = LongFlowNetCorr(f1_64,
f3_64_warp,
opt,
flow3_64,
trainable=trainable,
reuse=True,
namescope='flownet_64') + flow3_64
flow4_64 = LongFlowNetCorr(f1_64,
f4_64_warp,
opt,
flow4_64,
trainable=trainable,
reuse=True,
namescope='flownet_64') + flow4_64
flow5_64 = LongFlowNetCorr(f1_64,
f5_64_warp,
opt,
flow5_64,
trainable=trainable,
reuse=True,
namescope='flownet_64') + flow5_64
flow6_64 = LongFlowNetCorr(f1_64,
f6_64_warp,
opt,
flow6_64,
trainable=trainable,
reuse=True,
namescope='flownet_64') + flow6_64
f2_64_warp = WarpingLayer(f2_64, flow2_64)
f3_64_warp = WarpingLayer(f3_64, flow3_64)
f4_64_warp = WarpingLayer(f4_64, flow4_64)
f5_64_warp = WarpingLayer(f5_64, flow5_64)
f6_64_warp = WarpingLayer(f6_64, flow6_64)
f_stack_64_oth = tf.stack(
[f2_64_warp, f3_64_warp, f4_64_warp, f5_64_warp, f6_64_warp],
axis=2)
f_agg_64 = tf.squeeze(VI_Aggregator(f_stack_64_oth,
depth=5,
trainable=trainable,
namescope='st_agg_64'),
axis=2)
occlusion_mask_64 = MaskEstimator_(tf.abs(f1_64 - f_agg_64),
trainable=trainable,
namescope='masknet_64')
f_syn_64 = (1 -
occlusion_mask_64) * f1_64 + occlusion_mask_64 * f_agg_64
# ============================= SCALE - 1/2 : 128 ===============================
flow2_128 = resize_bilinear(flow2_64, (bot_h * 4, bot_w * 4),
align_corners=True) * 2
flow3_128 = resize_bilinear(flow3_64, (bot_h * 4, bot_w * 4),
align_corners=True) * 2
flow4_128 = resize_bilinear(flow4_64, (bot_h * 4, bot_w * 4),
align_corners=True) * 2
flow5_128 = resize_bilinear(flow5_64, (bot_h * 4, bot_w * 4),
align_corners=True) * 2
flow6_128 = resize_bilinear(flow6_64, (bot_h * 4, bot_w * 4),
align_corners=True) * 2
f2_128_warp = WarpingLayer(f2_128, flow2_128)
f3_128_warp = WarpingLayer(f3_128, flow3_128)
f4_128_warp = WarpingLayer(f4_128, flow4_128)
f5_128_warp = WarpingLayer(f5_128, flow5_128)
f6_128_warp = WarpingLayer(f6_128, flow6_128)
flow2_128 = LongFlowNetCorr(f1_128,
f2_128_warp,
opt,
flow2_128,
trainable=trainable,
reuse=False,
namescope='flownet_128') + flow2_128
flow3_128 = LongFlowNetCorr(f1_128,
f3_128_warp,
opt,
flow3_128,
trainable=trainable,
reuse=True,
namescope='flownet_128') + flow3_128
flow4_128 = LongFlowNetCorr(f1_128,
f4_128_warp,
opt,
flow4_128,
trainable=trainable,
reuse=True,
namescope='flownet_128') + flow4_128
flow5_128 = LongFlowNetCorr(f1_128,
f5_128_warp,
opt,
flow5_128,
trainable=trainable,
reuse=True,
namescope='flownet_128') + flow5_128
flow6_128 = LongFlowNetCorr(f1_128,
f6_128_warp,
opt,
flow6_128,
trainable=trainable,
reuse=True,
namescope='flownet_128') + flow6_128
f2_128_warp = WarpingLayer(f2_128, flow2_128)
f3_128_warp = WarpingLayer(f3_128, flow3_128)
f4_128_warp = WarpingLayer(f4_128, flow4_128)
f5_128_warp = WarpingLayer(f5_128, flow5_128)
f6_128_warp = WarpingLayer(f6_128, flow6_128)
f_stack_128_oth = tf.stack(
[f2_128_warp, f3_128_warp, f4_128_warp, f5_128_warp, f6_128_warp],
axis=2)
f_agg_128 = tf.squeeze(VI_Aggregator(f_stack_128_oth,
depth=5,
trainable=trainable,
namescope='st_agg_128'),
axis=2)
occlusion_mask_128 = MaskEstimator_(tf.abs(f1_128 - f_agg_128),
trainable=trainable,
namescope='masknet_128')
f_syn_128 = (
1 - occlusion_mask_128) * f1_128 + occlusion_mask_128 * f_agg_128
output, _ = VI_2D_Decoder_3(tf.expand_dims(state[0], axis=2),
opt,
x2_64_warp=tf.expand_dims(f_syn_64,
axis=2),
x2_128_warp=tf.expand_dims(f_syn_128,
axis=2),
trainable=trainable,
namescope='decoder')
occ_mask = resize_bilinear(occlusion_mask, (bot_h * 8, bot_w * 8),
align_corners=True)
occ_mask_64 = resize_bilinear(occlusion_mask_64,
(bot_h * 4, bot_w * 4),
align_corners=True)
occ_mask_128 = resize_bilinear(occlusion_mask_128,
(bot_h * 2, bot_w * 2),
align_corners=True)
flow6_256, flow6_512 = None, None
# NOTE: TF is static graph.
if opt.prev_warp:
if prev_state is not None or idx != 0:
h, w = flow6_128.get_shape().as_list()[2:4]
flow6_256 = resize_bilinear(flow6_128, (h * 2, w * 2)) * 2
flow6_512 = resize_bilinear(flow6_128, (h * 4, w * 4)) * 4
f6_256_warp = WarpingLayer(f6_256, flow6_256)
flow6_256 = LongFlowNetCorr(f1_256,
f6_256_warp,
opt,
flow6_256,
trainable=trainable,
namescope='flownet_256')
occlusion_mask_256 = MaskEstimator_(tf.abs(f1_256 -
f6_256_warp),
trainable=trainable,
namescope='masknet_256')
output_ = output
if opt.double_size:
prev_feed_warp = WarpingLayer(prev_feed[:, :3], flow6_512)
h, w = occlusion_mask_256.get_shape().as_list()[2:4]
occlusion_mask_512 = resize_nearest(
occlusion_mask_256, (h * 2, w * 2))
output = tf.squeeze(
(1 - tf.expand_dims(occlusion_mask_512, axis=2)) *
output + occlusion_mask_512 *
tf.expand_dims(prev_feed_warp, axis=2),
axis=2)
flow6_256 = flow6_512
else:
prev_feed_warp = WarpingLayer(prev_feed[:, :3], flow6_256)
output = tf.squeeze(
(1 - tf.expand_dims(occlusion_mask_256, axis=2)) *
output + occlusion_mask_256 *
tf.expand_dims(prev_feed_warp, axis=2),
axis=2)
if opt.loss_on_raw:
output = (output, output_)
# return output, tf.stack([flow2_128,flow3_128,flow4_128,flow5_128, flow6_128],axis=2), state, tf.stack([occ_mask, 1-occ_mask, occ_mask_64, 1-occ_mask_64, occ_mask_128, 1-occ_mask_128], axis=2), flow6_256
return output, state[0], state[1]
def call(self, inputs, cps=None, isFirstLayer=False):
if isFirstLayer:
tf.print("computed cp for the last input", cps[-1])
bOnA = inputs - self.firstLayerTA0 - cps / (self.firstLayerK1M *
self.E0)
tf.print("computed bOna for the last input", bOnA[-1])
tf.print("second parts",
(4 * inputs * cps / (self.firstLayerK1M * self.E0))[-1])
tf.assert_rank(
self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
tf.rank(inputs))
olderInput = tf.where(
tf.equal(
self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
0), inputs,
0.5 * (bOnA + (tf.pow(bOnA, 2) + 4 * inputs * cps /
(self.firstLayerK1M * self.E0))**0.5))
tf.print("solution for the last inputs of first layer: ",
olderInput[-1])
olderInput = self.firstLayerk2 * self.firstLayerK1M / self.firstLayerkdT * self.firstLayerTA0 * self.E0 / cps * olderInput / (
1 + self.firstLayerK1M * self.E0 / cps * olderInput)
else:
olderInput = inputs
#For batch computation: k1m * tf.transpose(inputs) doesn't work and we need to add a 1 axis in the end and use only *
#Just above we use rank1 vector so should keep rank2 batches of input
cpsExpand = tf.expand_dims(cps, -1)
tf.assert_rank(cpsExpand, 3)
tf.assert_rank(cps, 2)
olderInputExpand = tf.expand_dims(olderInput, -1)
tf.assert_rank(olderInputExpand, 3)
olderInputMidExpand = tf.expand_dims(olderInput, 1)
Cactivs = tf.where(
self.mask > 0, self.Cactiv /
(1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
Cinhibs = tf.where(
self.mask < 0, self.Cinhib /
(1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
tf.assert_rank(Cactivs, 3)
tf.assert_rank(Cinhibs, 3)
Inhib = tf.squeeze(tf.matmul(olderInputMidExpand, Cinhibs),
axis=1) / self.kdT
x_eq_clipped = tf.squeeze(tf.matmul(olderInputMidExpand, Cactivs),
axis=1) / (self.kdI * cps + Inhib / cps)
#unclipped version:
Cactivs_unclipped = tf.where(
self.kernel > 0, self.Cactiv * self.kernel /
(1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
Cinhibs_unclipped = tf.where(
self.kernel < 0, (-1) * self.Cinhib * self.kernel /
(1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
tf.assert_rank(Cactivs_unclipped, 3)
tf.assert_rank(Cinhibs_unclipped, 3)
#CAREFUL: now the cactivs has taken the batch size, it is of rank 3 : [None,inputdims,outputdims]
# THUS WE NEED: [None,1,inputdims] to use the matmul, and then squeeze the result!
Inhib_unclipped = tf.squeeze(tf.matmul(olderInputMidExpand,
Cinhibs_unclipped),
axis=1) / self.kdT
x_eq_unclipped = tf.squeeze(
tf.matmul(olderInputMidExpand, Cactivs_unclipped),
axis=1) / (self.kdI * cps + Inhib_unclipped / cps)
tf.assert_rank(tf.squeeze(tf.matmul(olderInputMidExpand,
Cinhibs_unclipped),
axis=1),
2,
message="compute not good dims")
tf.print("solution from an unknown layer:", x_eq_clipped[-1])
outputs = tf.stop_gradient(x_eq_clipped -
x_eq_unclipped) + x_eq_unclipped
tf.assert_rank(outputs, 2, message="outputs not good dims")
return outputs
def inception_v1(inputs,
num_classes=1000,
is_training=True,
dropout_keep_prob=0.8,
prediction_fn=slim.softmax,
spatial_squeeze=True,
reuse=None,
scope='InceptionV1',
global_pool=False):
"""Defines the Inception V1 architecture.
This architecture is defined in:
Going deeper with convolutions
Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,
Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.
http://arxiv.org/pdf/1409.4842v1.pdf.
The default image size used to train this network is 224x224.
Args:
inputs: a tensor of size [batch_size, height, width, channels].
num_classes: number of predicted classes. If 0 or None, the logits layer
is omitted and the input features to the logits layer (before dropout)
are returned instead.
is_training: whether is training or not.
dropout_keep_prob: the percentage of activation values that are retained.
prediction_fn: a function to get predictions out of logits.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is of
shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
global_pool: Optional boolean flag to control the avgpooling before the
logits layer. If false or unset, pooling is done with a fixed window
that reduces default-sized inputs to 1x1, while larger inputs lead to
larger outputs. If true, any input size is pooled down to 1x1.
Returns:
net: a Tensor with the logits (pre-softmax activations) if num_classes
is a non-zero integer, or the non-dropped-out input to the logits layer
if num_classes is 0 or None.
end_points: a dictionary from components of the network to the corresponding
activation.
"""
# Final pooling and prediction
with tf.variable_scope(scope, 'InceptionV1', [inputs], reuse=reuse) as scope:
with slim.arg_scope([slim.batch_norm, slim.dropout],
is_training=is_training):
net, end_points = inception_v1_base(inputs, scope=scope)
with tf.variable_scope('Logits'):
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2], keep_dims=True, name='global_pool')
end_points['global_pool'] = net
else:
# Pooling with a fixed kernel size.
net = slim.avg_pool2d(net, [7, 7], stride=1, scope='AvgPool_0a_7x7')
end_points['AvgPool_0a_7x7'] = net
if not num_classes:
return net, end_points
net = slim.dropout(net, dropout_keep_prob, scope='Dropout_0b')
logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None,
normalizer_fn=None, scope='Conv2d_0c_1x1')
if spatial_squeeze:
logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze')
end_points['Logits'] = logits
end_points['Predictions'] = prediction_fn(logits, scope='Predictions')
return logits, end_points
def inference(self):
self.input_x1_embed = tf.nn.embedding_lookup(self.Embedding,
self.input_x1)
self.input_x2_embed = tf.nn.embedding_lookup(self.Embedding,
self.input_x2)
with tf.variable_scope("lstm") as scope:
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_size)
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_size)
# with tf.variable_scope("rnn1"):
outputs1, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,
lstm_bw_cell,
self.input_x1_embed,
dtype=tf.float32)
outputs1_rnn = tf.concat(outputs1, axis=2)
self.outputs1_last = outputs1_rnn[:, -1, :]
scope.reuse_variables()
outputs2, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,
lstm_bw_cell,
self.input_x2_embed,
dtype=tf.float32)
outputs2_rnn = tf.concat(outputs2, axis=2)
self.outputs2_last = outputs2_rnn[:, -1, :]
with tf.variable_scope("outputs"):
# self.conc = tf.concat([self.outputs1_last, self.outputs2_last], axis=1)
self.manha = tf.reduce_sum(tf.abs(
tf.subtract(self.outputs1_last, self.outputs2_last)),
axis=1,
keepdims=True)
self.squre = tf.reduce_sum(tf.square(
tf.subtract(self.outputs1_last, self.outputs2_last)),
axis=1,
keepdims=True)
self.element_wise = tf.multiply(self.outputs1_last,
self.outputs2_last)
self.norm1 = tf.sqrt(
tf.reduce_sum(tf.square(self.outputs1_last),
axis=1,
keepdims=True))
self.norm2 = tf.sqrt(
tf.reduce_sum(tf.square(self.outputs2_last),
axis=1,
keepdims=True))
self.sum12 = tf.reduce_sum(self.element_wise,
axis=1,
keepdims=True)
self.cos = tf.divide(self.sum12,
tf.multiply(self.norm1, self.norm2))
self.input_dense = tf.concat([
self.element_wise, self.manha, self.norm1, self.norm2,
self.sum12, self.cos, self.squre
],
axis=1)
self.fc1 = tf.layers.dense(self.input_dense,
256,
activation=tf.nn.relu)
self.fc1 = tf.nn.dropout(self.fc1,
keep_prob=self.dropout_keep_prob)
self.fc2 = tf.layers.dense(self.fc1, 128, activation=tf.nn.relu)
self.fc2 = tf.nn.dropout(self.fc2,
keep_prob=self.dropout_keep_prob)
self.fc3 = tf.layers.dense(self.fc2, 32, activation=tf.nn.relu)
self.fc3 = tf.nn.dropout(self.fc3,
keep_prob=self.dropout_keep_prob)
self.logits = tf.squeeze(tf.layers.dense(self.fc3,
1,
activation=tf.nn.sigmoid),
axis=1,
name="predict")
return self.logits
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.int32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="annotation")
pred_annotation, logits = inference(image, keep_probability)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
tf.summary.image("pred_annotation",
tf.cast(pred_annotation, tf.uint8),
max_outputs=2)
loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=tf.squeeze(annotation, squeeze_dims=[3]),
name="entropy")))
tf.summary.scalar("entropy", loss)
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
#print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
#validation_dataset_reader = dataset.BatchDatset(valid_records, image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, sess.graph)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, summary_op],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
summary_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
# valid_images, valid_annotations = validation_dataset_reader.next_batch(FLAGS.batch_size)
# valid_loss = sess.run(loss, feed_dict={image: valid_images, annotation: valid_annotations,
# keep_probability: 1.0})
# print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = train_dataset_reader.get_random_batch(
FLAGS.batch_size)
pred = sess.run(pred_annotation,
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0
})
valid_annotations = np.squeeze(valid_annotations, axis=3)
pred = np.squeeze(pred, axis=3)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(valid_annotations[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="pred_" + str(5 + itr))
print("Saved image: %d" % itr)
env = ArmEnv(size_x=4,
size_y=3,
cubes_cnt=4,
episode_max_length=2000,
finish_reward=200,
action_minus_reward=0.0,
tower_target_size=3) #is 0.1 IMPORTANT???????????
s = env.reset()
obs_len = len(s)
tf.reset_default_graph()
state = tf.placeholder('float32', shape=[None, obs_len], name="STATE")
actions = tf.squeeze(tf.placeholder('int32', name="ACTIONS"))
'''
============================
Actor = Policy Approxiamtion
============================
'''
q_estimation = tf.placeholder('float32', name="Q-Estimation")
inp = tf.layers.dense(state,
10,
name="INPUT",
kernel_initializer=tf.random_normal_initializer(
mean=0, stddev=0.1),
bias_initializer=tf.initializers.constant(0))
out = tf.layers.dense(inp,
def build_model(self):
# Kernel initialization for the convolutions
self.init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.02)
# Placeholders
self.is_training = tf.placeholder(tf.bool)
self.image_input = tf.placeholder(tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="x")
self.noise_tensor = tf.placeholder(
tf.float32,
shape=[None, self.config.trainer.noise_dim],
name="noise")
self.true_labels = tf.placeholder(dtype=tf.float32,
shape=[None, 1],
name="true_labels")
self.generated_labels = tf.placeholder(dtype=tf.float32,
shape=[None, 1],
name="gen_labels")
self.real_noise = tf.placeholder(dtype=tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="real_noise")
self.fake_noise = tf.placeholder(dtype=tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="fake_noise")
self.logger.info("Building training graph...")
with tf.variable_scope("BIGAN"):
with tf.variable_scope("Encoder_Model"):
self.noise_gen = self.encoder(self.image_input)
with tf.variable_scope("Generator_Model"):
self.image_gen = self.generator(
self.noise_tensor) + self.fake_noise
self.reconstructed = self.generator(self.noise_gen)
with tf.variable_scope("Discriminator_Model"):
# E(x) and x --> This being real is the output of discriminator
l_encoder, inter_layer_inp = self.discriminator(
self.noise_gen, self.image_input + self.real_noise)
# z and G(z)
l_generator, inter_layer_rct = self.discriminator(
self.noise_tensor, self.image_gen)
# Loss Function Implementations
with tf.name_scope("Loss_Functions"):
# Discriminator
# Discriminator sees the encoder result as true because it discriminates E(x), x as the real pair
self.loss_dis_enc = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.true_labels, logits=l_encoder))
self.loss_dis_gen = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.generated_labels, logits=l_generator))
self.loss_discriminator = self.loss_dis_enc + self.loss_dis_gen
if self.config.trainer.flip_labels:
labels_gen = tf.zeros_like(l_generator)
labels_enc = tf.ones_like(l_encoder)
else:
labels_gen = tf.ones_like(l_generator)
labels_enc = tf.zeros_like(l_encoder)
# Generator
# Generator is considered as the true ones here because it tries to fool discriminator
self.loss_generator = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_gen,
logits=l_generator))
# Encoder
# Encoder is considered as the fake one because it tries to fool the discriminator also
self.loss_encoder = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_enc,
logits=l_encoder))
# Optimizer Implementations
with tf.name_scope("Optimizers"):
# Build the optimizers
self.generator_optimizer = tf.train.AdamOptimizer(
self.config.trainer.generator_l_rate,
beta1=self.config.trainer.optimizer_adam_beta1,
beta2=self.config.trainer.optimizer_adam_beta2,
)
self.discriminator_optimizer = tf.train.AdamOptimizer(
self.config.trainer.discriminator_l_rate,
beta1=self.config.trainer.optimizer_adam_beta1,
beta2=self.config.trainer.optimizer_adam_beta2,
)
self.encoder_optimizer = tf.train.AdamOptimizer(
self.config.trainer.generator_l_rate,
beta1=self.config.trainer.optimizer_adam_beta1,
beta2=self.config.trainer.optimizer_adam_beta2,
)
# Collect all the variables
all_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
# Generator Network Variables
self.generator_vars = [
v for v in all_variables
if v.name.startswith("BIGAN/Generator_Model")
]
# Discriminator Network Variables
self.discriminator_vars = [
v for v in all_variables
if v.name.startswith("BIGAN/Discriminator_Model")
]
# Encoder Network Variables
self.encoder_vars = [
v for v in all_variables
if v.name.startswith("BIGAN/Encoder_Model")
]
# Create Training Operations
# Generator Network Operations
self.gen_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Generator_Model")
# Discriminator Network Operations
self.disc_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Discriminator_Model")
# Encoder Network Operations
self.enc_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Encoder_Model")
# Initialization of Optimizers
with tf.control_dependencies(self.gen_update_ops):
self.gen_op = self.generator_optimizer.minimize(
self.loss_generator,
var_list=self.generator_vars,
global_step=self.global_step_tensor,
)
with tf.control_dependencies(self.disc_update_ops):
self.disc_op = self.discriminator_optimizer.minimize(
self.loss_discriminator, var_list=self.discriminator_vars)
with tf.control_dependencies(self.enc_update_ops):
self.enc_op = self.encoder_optimizer.minimize(
self.loss_encoder, var_list=self.encoder_vars)
# Exponential Moving Average for Estimation
self.dis_ema = tf.train.ExponentialMovingAverage(
decay=self.config.trainer.ema_decay)
maintain_averages_op_dis = self.dis_ema.apply(
self.discriminator_vars)
self.gen_ema = tf.train.ExponentialMovingAverage(
decay=self.config.trainer.ema_decay)
maintain_averages_op_gen = self.gen_ema.apply(self.generator_vars)
self.enc_ema = tf.train.ExponentialMovingAverage(
decay=self.config.trainer.ema_decay)
maintain_averages_op_enc = self.enc_ema.apply(self.encoder_vars)
with tf.control_dependencies([self.disc_op]):
self.train_dis_op = tf.group(maintain_averages_op_dis)
with tf.control_dependencies([self.gen_op]):
self.train_gen_op = tf.group(maintain_averages_op_gen)
with tf.control_dependencies([self.enc_op]):
self.train_enc_op = tf.group(maintain_averages_op_enc)
self.logger.info("Building Testing Graph...")
with tf.variable_scope("BIGAN"):
with tf.variable_scope("Encoder_Model"):
self.noise_gen_ema = self.encoder(self.image_input,
getter=get_getter(
self.enc_ema))
with tf.variable_scope("Generator_Model"):
self.reconstruct_ema = self.generator(self.noise_gen_ema,
getter=get_getter(
self.gen_ema))
with tf.variable_scope("Discriminator_Model"):
self.l_encoder_ema, self.inter_layer_inp_ema = self.discriminator(
self.noise_gen_ema, # E(x)
self.image_input, # x
getter=get_getter(self.dis_ema),
)
self.l_generator_ema, self.inter_layer_rct_ema = self.discriminator(
self.noise_gen_ema, # E(x)
self.reconstruct_ema, # G(E(x))
getter=get_getter(self.dis_ema),
)
with tf.name_scope("Testing"):
with tf.variable_scope("Reconstruction_Loss"):
# LG(x) = ||x - G(E(x))||_1
delta = self.image_input - self.reconstruct_ema
delta_flat = tf.layers.Flatten()(delta)
self.gen_score = tf.norm(
delta_flat,
ord=self.config.trainer.degree,
axis=1,
keepdims=False,
name="epsilon",
)
with tf.variable_scope("Discriminator_Loss"):
if self.config.trainer.loss_method == "cross_e":
self.dis_score = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(self.l_encoder_ema),
logits=self.l_encoder_ema)
elif self.config.trainer.loss_method == "fm":
fm = self.inter_layer_inp_ema - self.inter_layer_rct_ema
fm = tf.layers.Flatten()(fm)
self.dis_score = tf.norm(fm,
ord=self.config.trainer.degree,
axis=1,
keepdims=False,
name="d_loss")
self.dis_score = tf.squeeze(self.dis_score)
with tf.variable_scope("Score"):
self.list_scores = (
1 - self.config.trainer.weight
) * self.dis_score + self.config.trainer.weight * self.gen_score
if self.config.trainer.enable_early_stop:
self.rec_error_valid = tf.reduce_mean(self.list_scores)
if self.config.log.enable_summary:
with tf.name_scope("Summary"):
with tf.name_scope("Disc_Summary"):
tf.summary.scalar("loss_discriminator",
self.loss_discriminator, ["dis"])
tf.summary.scalar("loss_dis_encoder", self.loss_dis_enc,
["dis"])
tf.summary.scalar("loss_dis_gen", self.loss_dis_gen,
["dis"])
with tf.name_scope("Gen_Summary"):
tf.summary.scalar("loss_generator", self.loss_generator,
["gen"])
tf.summary.scalar("loss_encoder", self.loss_encoder,
["gen"])
with tf.name_scope("Image_Summary"):
tf.summary.image("reconstruct", self.reconstructed, 3,
["image"])
tf.summary.image("input_images", self.image_input, 3,
["image"])
if self.config.trainer.enable_early_stop:
with tf.name_scope("validation_summary"):
tf.summary.scalar("valid", self.rec_error_valid, ["v"])
self.sum_op_dis = tf.summary.merge_all("dis")
self.sum_op_gen = tf.summary.merge_all("gen")
self.sum_op_im = tf.summary.merge_all("image")
self.sum_op_valid = tf.summary.merge_all("v")