def testGradSerialTwoLoops(self):
with self.test_session():
num_steps = 100
acc = tf.TensorArray(dtype=tf.float32, size=num_steps,
clear_after_read=False,
element_shape=tensor_shape.scalar())
i = tf.constant(0, name="i")
x = tf.constant(2.0, name="x")
c = lambda i, acc: i < 5
def b(i, acc):
x1 = tf.cond(tf.equal(i, 0),
lambda: x,
lambda: tf.mul(acc.read(i - 1), 2.0))
return i + 1, acc.write(i, x1)
i1, acc1 = tf.while_loop(c, b, [i, acc])
z = tf.constant(0.0)
def fn(i, acc):
return i + 1, acc.write(i, z)
_, acc2 = tf.while_loop(lambda i, acc: i < num_steps, fn, [i1, acc1])
r = acc2.stack()
grad = tf.gradients(r, [x])[0]
self.assertAllClose(31.0, grad.eval())
def _testStackWhileSwap(self, use_gpu):
with self.test_session(use_gpu=use_gpu):
n = tf.constant(0)
h = gen_data_flow_ops._stack(tf.float32, stack_name="foo")
def c(x):
return tf.less(x, 10)
def b(x):
with tf.control_dependencies([x]):
a = tf.constant(np.ones(2000), dtype=tf.float32)
v = gen_data_flow_ops._stack_push(h, a, swap_memory=True)
with tf.control_dependencies([v]):
return tf.add(x, 1)
r = tf.while_loop(c, b, [n])
v = tf.constant(np.zeros(2000), dtype=tf.float32)
def c1(x, y):
return tf.greater(x, 0)
def b1(x, y):
nx = tf.sub(x, 1)
ny = y + gen_data_flow_ops._stack_pop(h, tf.float32)
return [nx, ny]
rx, ry = tf.while_loop(c1, b1, [r, v],
[r.get_shape(), tensor_shape.unknown_shape()])
self.assertAllClose(np.ones(2000) * 10.0, ry.eval())
def body(x):
def nest_body(c):
return tf.multiply(c, 2)
def cd(c): return tf.less(c, 10)
c = tf.constant(2)
res = tf.while_loop(cd, nest_body, loop_vars=[c])
return tf.nn.relu(x + res)
def batch_embed_lookup(embedding, ids):
'''
embedding: shape(b_sz, tstp, emb_sz)
ids : shape(b_sz, k)
'''
input_shape = tf.shape(embedding)
time_steps = input_shape[0]
def _create_ta(name, dtype):
return tf.TensorArray(dtype=dtype,
size=time_steps,
tensor_array_name=name)
input_ta = _create_ta('input_ta', embedding.dtype)
fetch_ta = _create_ta('fetch_ta', ids.dtype)
output_ta = _create_ta('output_ta', embedding.dtype)
input_ta = input_ta.unstack(embedding)
fetch_ta = fetch_ta.unstack(ids)
def loop_body(time, output_ta):
embed = input_ta.read(time) #shape(tstp, emb_sz) type of float32
fetch_id = fetch_ta.read(time) #shape(tstp) type of int32
out_emb = tf.nn.embedding_lookup(embed, fetch_id)
output_ta = output_ta.write(time, out_emb)
next_time = time+1
return next_time, output_ta
time = tf.constant(0)
_, output_ta = tf.while_loop(cond=lambda time, *_: time < time_steps,
body=loop_body, loop_vars=(time, output_ta),
swap_memory=True)
ret_t = output_ta.stack() #shape(b_sz, tstp, embd_sz)
return ret_t
def loss(self, predicts, labels, objects_num):
"""Add Loss to all the trainable variables
Args:
predicts: 4-D tensor [batch_size, cell_size, cell_size, 5 * boxes_per_cell]
===> (num_classes, boxes_per_cell, 4 * boxes_per_cell)
labels : 3-D tensor of [batch_size, max_objects, 5]
objects_num: 1-D tensor [batch_size]
"""
class_loss = tf.constant(0, tf.float32)
object_loss = tf.constant(0, tf.float32)
noobject_loss = tf.constant(0, tf.float32)
coord_loss = tf.constant(0, tf.float32)
loss = [0, 0, 0, 0]
for i in range(self.batch_size):
predict = predicts[i, :, :, :]
label = labels[i, :, :]
object_num = objects_num[i]
nilboy = tf.ones([7,7,2])
tuple_results = tf.while_loop(self.cond1, self.body1, [tf.constant(0), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy])
for j in range(4):
loss[j] = loss[j] + tuple_results[2][j]
nilboy = tuple_results[5]
tf.add_to_collection('losses', (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size)
tf.summary.scalar('class_loss', loss[0]/self.batch_size)
tf.summary.scalar('object_loss', loss[1]/self.batch_size)
tf.summary.scalar('noobject_loss', loss[2]/self.batch_size)
tf.summary.scalar('coord_loss', loss[3]/self.batch_size)
tf.summary.scalar('weight_loss', tf.add_n(tf.get_collection('losses')) - (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size )
return tf.add_n(tf.get_collection('losses'), name='total_loss'), nilboy
def prior_model(prior_queue_init, length=SIG_LEN):
def cond(loop_counter, *_):
return tf.less(loop_counter, length)
def body(loop_counter, accumulated_output_array, accumulated_logits_array, next_input, *queue_contents):
next_logit, queue_updates = sub_predictor(next_input, queue_contents)
gumbeled = next_logit[:, 0, :] - tf.log(-tf.log(tf.random_uniform((tf.shape(next_logit)[0], QUANT_LEVELS))))
sample_disc = tf.arg_max(gumbeled, 1)
sample_cont = dequantizer(sample_disc, QUANT_LOWER, QUANT_UPPER, QUANT_LEVELS)
accumulated_output_array = accumulated_output_array.write(loop_counter, sample_cont)
accumulated_logits_array = accumulated_logits_array.write(loop_counter, next_logit[:, 0, :])
sample_cont = tf.expand_dims(sample_cont, 1)
sample_cont = tf.expand_dims(sample_cont, 1) # sic
next_input = tf.concat(2, (sample_cont, tf.ones_like(sample_cont)))
return [loop_counter+1, accumulated_output_array, accumulated_logits_array, next_input] + queue_updates
accumulated_output_array = tf.TensorArray(tf.float32, size=SIG_LEN, clear_after_read=False)
accumulated_logits_array = tf.TensorArray(tf.float32, size=SIG_LEN, clear_after_read=False)
loop_var_init = [tf.constant(0, dtype=tf.int32), accumulated_output_array, accumulated_logits_array, tf.zeros((PRIOR_BATCH_SIZE, 1, 2))] + prior_queue_init
accumulated_output_array, accumulated_logits_array = tf.while_loop(cond, body, loop_var_init, back_prop=False)[1:3]
output = tf.transpose(accumulated_output_array.pack(), [1, 0])
logits = tf.transpose(accumulated_logits_array.pack(), [1, 0, 2])
output.set_shape((PRIOR_BATCH_SIZE, length))
logits.set_shape((PRIOR_BATCH_SIZE, length, QUANT_LEVELS))
return output, logits
def forward_prop_nodes(i_start, size, acts, offset):
# Note: In the corpus that we've seen, parse trees are always ordered such that
# iteration forward through the list will be in bottom-up order.
# Conversely, iteration in reverse is always top-down.
# This enables a simple iterative algorithm. If this were not the case,
# putting the nodes in order by a postorder traversal would fix it.
def fwd_continue(*parms):
(_, sz, cur, _) = parms
return tf.less(cur, sz, name='cur_le_size')
def forward_prop(*parms):
(i0, sz, cur, act) = parms
with tf.device('/gpu:0'):
gact = act
gcur = cur
next_idx = i0 + gcur
node_out = tf.reshape(forward_node(next_idx, act, offset), [1, FLAGS.wvs, 1], name='node_out')
tf.scatter_add(gact, tf.pack([gcur]), node_out, name='act_update')
act = gact
return [i0, sz, cur + iONE, act]
with tf.device('/cpu:0'):
i_start = tf.convert_to_tensor(i_start, dtype=tf.int32, name='i_start')
size = tf.convert_to_tensor(size, dtype=tf.int32, name='size')
iZ = tf.convert_to_tensor(0, dtype=tf.int32, name='ZERO')
while_parms = [i_start, size, iZ, acts]
wresult = tf.while_loop(fwd_continue, forward_prop, while_parms, parallel_iterations=1,
name='forward_prop_while')
(_, _, _, result) = wresult
return tf.slice(result, [0, 0, 0], tf.pack([size, -1, -1]), name='fwd_prop_nodes')
def hinge_loss(self, y_true, y_pred):
# Custom loss function
margin = 0.1
# loop counter
i = tf.constant(0)
# loop condition function
c = lambda i, _: tf.less(i, tf.shape(y_true)[0])
outer_sum_loss = tf.constant(0.0)
def process_ele(i, outer_sum_loss):
# Get a subtensor from batch
y_true_one = y_true[i]
y_pred_one = y_pred[i]
# Stack margin to a num_class*1 matrix
margin_stack = tf.reshape(tf.stack([tf.constant(0.1)] * self.num_classes), [self.num_classes, 1])
# Stack true label to a word_dim*num_class matrix and transpose it
y_true_one_stack = tf.stack([tf.transpose(y_true_one)] * self.num_classes)
# Reshape predict from (word_dim,) to (word_dim,1)
y_pred_one_t = tf.reshape(y_pred_one, [self.word_dim, 1])
# Calculate loss
r = margin_stack - tf.matmul(y_true_one_stack, y_pred_one_t) + tf.matmul(self.label_vec_tensor, y_pred_one_t)
# Summation
# We did not exclude true label inside summation, so we subtract extra margin
sum_inner_loss = tf.reduce_sum(K.relu(r)) - margin
# Return counter++ and accumulated loss
return tf.add(i, 1), tf.add(outer_sum_loss, sum_inner_loss)
_, outer_sum_loss = tf.while_loop(c, process_ele, [i, outer_sum_loss])
# Return average loss over batch
return outer_sum_loss / tf.cast(tf.shape(y_true)[0], dtype=tf.float32)
def decoding_loop(self) -> BeamSearchOutput:
"""Create the decoding loop.
This function mimics the behavior of the ``decoding_loop`` method of
the ``AutoregressiveDecoder``, except the initial loop state is created
outside this method because it is accessed and fed during ensembling.
TODO: The ``finalize_loop`` method and the handling of attention loop
states might be implemented in the future.
Returns:
This method returns a populated ``BeamSearchOutput`` object.
"""
final_loop_state = tf.while_loop(
self.loop_continue_criterion,
self.get_body(),
self.initial_loop_state,
shape_invariants=tf.contrib.framework.nest.map_structure(
get_state_shape_invariants, self.initial_loop_state))
# TODO: return att_loop_states properly
return BeamSearchOutput(
last_search_step_output=final_loop_state.search_results,
last_dec_loop_state=final_loop_state.decoder_loop_state,
last_search_state=final_loop_state.search_state,
attention_loop_states=[])
def _build_train_specific_graph(self, iterator, model_fn, params,
record_files_placeholder, num_train_steps):
"""Builds the part of the model that is specific to training."""
def build():
features, labels = iterator.get_next()
estimator_spec = model_fn(
features, labels, tf.estimator.ModeKeys.TRAIN, params)
with tf.control_dependencies([estimator_spec.train_op]):
run_model_op = self._global_step.assign_add(1)
return run_model_op, estimator_spec.loss
if self._use_while_loop:
def body(i):
run_model_op_single_step, _ = build()
with tf.control_dependencies([run_model_op_single_step]):
return i + 1
run_model_op = tf.while_loop(lambda i: i < num_train_steps, body, [0],
parallel_iterations=1)
loss = None
else:
run_model_op, loss = build()
return self._TrainModelProperties(
record_files_placeholder, iterator, loss, params["batch_size"],
run_model_op)
def batch_logits(indices, acts):
init_outs = tf.zeros([1, FLAGS.wvs, 1])
def logits_continue(*parms):
cur, idxs, _, _, _ = parms
return tf.less(cur, tf.size(idxs), name='batch_done')
def logits_batch_body(*parms):
i, idxs, ptr, css, act = parms
i_s = tf.reshape(tf.slice(idxs, tf.pack([i]), [1]), [])
start, size = get_bounds(i_s)
outs = forward_prop_nodes(start, size, acts, ptr)
new_css = tf.cond(tf.equal(i, iZERO),
lambda: outs,
lambda: tf.concat(0, [css, outs]))
return i + iONE, indices, ptr + size, new_css, acts
with tf.device('/cpu:0'):
iZ = tf.convert_to_tensor(0, dtype=tf.int32)
zero_activations(acts)
while_parms = [iZ, indices, iZ, init_outs, acts]
_, _, _, outs, _ = tf.while_loop(logits_continue, logits_batch_body, while_parms,
parallel_iterations=1, name='batch_logits')
lumpy_logits = tf.map_fn(activation_to_logits, outs, name='raw_logits')
logits = tf.squeeze(lumpy_logits, [2], name='logits')
return logits
def decoding_loop(self, train_mode: bool, sample: bool = False,
temperature: float = 1) -> LoopState:
"""Run the decoding while loop.
Calls get_initial_loop_state and constructs tf.while_loop
with the continuation criterion returned from loop_continue_criterion,
and body function returned from get_body.
After finishing the tf.while_loop, it calls finalize_loop
to further postprocess the final decoder loop state (usually
by stacking Tensors containing decoding histories).
Arguments:
train_mode: Boolean flag, telling whether this is
a training run.
sample: Boolean flag, telling whether we should sample
the output symbols from the output distribution instead
of using argmax or gold data.
temperature: float value specifying the softmax temperature
"""
initial_loop_state = self.get_initial_loop_state()
with tf.control_dependencies([self.decoding_w, self.decoding_b]):
final_loop_state = tf.while_loop(
self.loop_continue_criterion,
self.get_body(train_mode, sample, temperature),
initial_loop_state,
shape_invariants=tf.contrib.framework.nest.map_structure(
get_state_shape_invariants, initial_loop_state))
self.finalize_loop(final_loop_state, train_mode)
return final_loop_state
def _build_eval_specific_graph(self, iterator, model_fn, params,
record_files_placeholder, num_eval_steps):
"""Builds the part of the model that is specific to evaluation."""
def build():
features = iterator.get_next()
estimator_spec = model_fn(
features, None, tf.estimator.ModeKeys.EVAL, params)
run_model_op = tf.group(*(update_op for _, update_op in
estimator_spec.eval_metric_ops.values()))
eval_metric_tensors = {k: tensor for (k, (tensor, _))
in estimator_spec.eval_metric_ops.items()}
return run_model_op, estimator_spec.loss, eval_metric_tensors
if self._use_while_loop:
def body(i):
run_model_op_single_step, _, _ = build()
with tf.control_dependencies([run_model_op_single_step]):
return i + 1
run_model_op = tf.while_loop(lambda i: i < num_eval_steps, body, [0],
parallel_iterations=1)
loss = None
eval_metric_tensors = {
"HR": self._compute_metric_mean(rconst.HR_METRIC_NAME),
"NDCG": self._compute_metric_mean(rconst.NDCG_METRIC_NAME),
}
else:
run_model_op, loss, eval_metric_tensors = build()
metric_initializer = tf.variables_initializer(
tf.get_collection(tf.GraphKeys.METRIC_VARIABLES))
return self._EvalModelProperties(
record_files_placeholder, iterator, loss, params["eval_batch_size"],
run_model_op, eval_metric_tensors, metric_initializer)
def _sample_n(self, n, seed=None):
"""Sample `n` draws from the DP. Draws from the base
distribution are memoized across `n` and across calls to
`sample()`.
Draws from the base distribution are not memoized across the batch
shape, i.e., each independent DP in the batch shape has its own
memoized samples.
Returns:
tf.Tensor.
A `tf.Tensor` of shape `[n] + batch_shape + event_shape`,
where `n` is the number of samples for each DP,
`batch_shape` is the number of independent DPs, and
`event_shape` is the shape of the base distribution.
#### Notes
The implementation has one inefficiency, which is that it draws
(batch_shape,) samples from the base distribution when adding a
new persistent state. Ideally, we would only draw new samples for
those in the loop which require it.
"""
if seed is not None:
raise NotImplementedError("seed is not implemented.")
batch_shape = self.batch_shape.as_list()
event_shape = self.event_shape.as_list()
rank = 1 + len(batch_shape) + len(event_shape)
# Note this is for scoping within the while loop's body function.
self._temp_scope = [n, batch_shape, event_shape, rank]
# Start at the beginning of the stick, i.e. the k'th index
k = tf.constant(0)
# Define boolean tensor. It is True for samples that require continuing
# the while loop and False for samples that can receive their base
# distribution (coin lands heads). Also note that we need one bool for
# each sample
bools = tf.ones([n] + batch_shape, dtype=tf.bool)
# Initialize all samples as zero, they will be overwritten in any case
draws = tf.zeros([n] + batch_shape + event_shape, dtype=self.base.dtype)
# Calculate shape invariance conditions for locs and probs as these
# can change shape between loop iterations.
locs_shape = tf.TensorShape([None])
probs_shape = tf.TensorShape([None])
if len(self.locs.shape) > 1:
locs_shape = locs_shape.concatenate(self.locs.shape[1:])
probs_shape = probs_shape.concatenate(self.probs.shape[1:])
# While we have not broken enough sticks, keep sampling.
_, _, self._locs, self._probs, samples = tf.while_loop(
self._sample_n_cond, self._sample_n_body,
loop_vars=[k, bools, self.locs, self.probs, draws],
shape_invariants=[
k.shape, bools.shape, locs_shape, probs_shape, draws.shape])
return samples
def geometric(p):
i = tf.constant(0)
sample = tf.while_loop(
cond=lambda i: tf.cast(1 - Bernoulli(probs=p), tf.bool),
body=lambda i: i + 1,
loop_vars=[i])
return sample
def compute_states(self,emb,idx_batch=0):
num_leaves = tf.squeeze(tf.gather(self.num_leaves,idx_batch))
#num_leaves=tf.Print(num_leaves,[num_leaves])
n_inodes = tf.gather(self.n_inodes,idx_batch)
#embx=tf.gather(emb,tf.range(num_leaves))
embx=tf.gather(tf.gather(emb,idx_batch),tf.range(num_leaves))
#treestr=self.treestr#tf.gather(self.treestr,tf.range(self.n_inodes))
treestr=tf.gather(tf.gather(self.treestr,idx_batch),tf.range(n_inodes))
leaf_hc = self.process_leafs(embx)
leaf_h,leaf_c=tf.split(1,2,leaf_hc)
node_h=tf.identity(leaf_h)
node_c=tf.identity(leaf_c)
idx_var=tf.constant(0) #tf.Variable(0,trainable=False)
with tf.variable_scope("Composition",reuse=True):
cW = tf.get_variable("cW",[self.degree*self.hidden_dim,(self.degree+3)*self.hidden_dim])
cb = tf.get_variable("cb",[4*self.hidden_dim])
bu,bo,bi,bf=tf.split(0,4,cb)
def _recurrence(node_h,node_c,idx_var):
node_info=tf.gather(treestr,idx_var)
child_h=tf.gather(node_h,node_info)
child_c=tf.gather(node_c,node_info)
flat_ = tf.reshape(child_h,[-1])
tmp=tf.matmul(tf.expand_dims(flat_,0),cW)
u,o,i,fl,fr=tf.split(1,5,tmp)
i=tf.nn.sigmoid(i+bi)
o=tf.nn.sigmoid(o+bo)
u=tf.nn.tanh(u+bu)
fl=tf.nn.sigmoid(fl+bf)
fr=tf.nn.sigmoid(fr+bf)
f=tf.concat(0,[fl,fr])
c = i * u + tf.reduce_sum(f*child_c,[0])
h = o * tf.nn.tanh(c)
node_h = tf.concat(0,[node_h,h])
node_c = tf.concat(0,[node_c,c])
idx_var=tf.add(idx_var,1)
return node_h,node_c,idx_var
loop_cond = lambda a1,b1,idx_var: tf.less(idx_var,n_inodes)
loop_vars=[node_h,node_c,idx_var]
node_h,node_c,idx_var=tf.while_loop(loop_cond, _recurrence,
loop_vars,parallel_iterations=10)
return node_h
def while_module_fn():
"""Compute x^n with while_loop."""
x = tf.placeholder(dtype=tf.float32, name="x", shape=[])
n = tf.placeholder(dtype=tf.int32, name="n")
_, pow_x = tf.while_loop(
lambda i, ix: i < n, lambda i, ix: [tf.add(i, 1), ix * x],
[tf.constant(0), tf.constant(1.0)])
hub.add_signature(inputs={"x": x, "n": n}, outputs=pow_x)
def fn1(a, b):
i = tf.constant(0)
def cd(i): return tf.less(i, 10)
def bd(i): return tf.add(i, 1)
res = tf.while_loop(cd, bd, [i])
return tf.multiply(tf.add(20, res), 10)
def custom_dynamic_rnn(cell, inputs, inputs_len, initial_state=None):
"""
Implements a dynamic rnn that can store scores in the pointer network,
the reason why we implements this is that the raw_rnn or dynamic_rnn function in Tensorflow
seem to require the hidden unit and memory unit has the same dimension, and we cannot
store the scores directly in the hidden unit.
Args:
cell: RNN cell
inputs: the input sequence to rnn
inputs_len: valid length
initial_state: initial_state of the cell
Returns:
outputs and state
"""
batch_size = tf.shape(inputs)[0]
max_time = tf.shape(inputs)[1]
inputs_ta = tf.TensorArray(dtype=tf.float32, size=max_time)
inputs_ta = inputs_ta.unstack(tf.transpose(inputs, [1, 0, 2]))
emit_ta = tf.TensorArray(dtype=tf.float32, dynamic_size=True, size=0)
t0 = tf.constant(0, dtype=tf.int32)
if initial_state is not None:
s0 = initial_state
else:
s0 = cell.zero_state(batch_size, dtype=tf.float32)
f0 = tf.zeros([batch_size], dtype=tf.bool)
def loop_fn(t, prev_s, emit_ta, finished):
"""
the loop function of rnn
"""
cur_x = inputs_ta.read(t)
scores, cur_state = cell(cur_x, prev_s)
# copy through
scores = tf.where(finished, tf.zeros_like(scores), scores)
if isinstance(cell, tc.rnn.LSTMCell):
cur_c, cur_h = cur_state
prev_c, prev_h = prev_s
cur_state = tc.rnn.LSTMStateTuple(tf.where(finished, prev_c, cur_c),
tf.where(finished, prev_h, cur_h))
else:
cur_state = tf.where(finished, prev_s, cur_state)
emit_ta = emit_ta.write(t, scores)
finished = tf.greater_equal(t + 1, inputs_len)
return [t + 1, cur_state, emit_ta, finished]
_, state, emit_ta, _ = tf.while_loop(
cond=lambda _1, _2, _3, finished: tf.logical_not(tf.reduce_all(finished)),
body=loop_fn,
loop_vars=(t0, s0, emit_ta, f0),
parallel_iterations=32,
swap_memory=False)
outputs = tf.transpose(emit_ta.stack(), [1, 0, 2])
return outputs, state
def construct_tensor_array(self):
loop_condition = lambda tensor_array, i: \
tf.less(i, tf.squeeze(tf.shape(self.is_a_leaf)))
#iterate over all leaves + composition
tensor_array_op = tf.while_loop(cond=loop_condition,
body=self._loop_over_tree,
loop_vars=[self.tensor_array, 0],
parallel_iterations=1)[0]
return tensor_array_op
def tf_orthogonalize(W, eps=1e-6, back_prop=False):
p = W.get_shape()[0].value
eye = tf.constant(np.eye(p, dtype=np.float32))
def ortho_step(Q0):
Q1 = Q0 / tf.sqrt(tf_max_abs_row_sum_norm(tf.matmul(Q0, Q0, transpose_b=True)))
return 3/2*Q1 - 0.5*tf.matmul(Q1, tf.matmul(Q1, Q1, transpose_a=True))
c = lambda Q: tf.greater(tf_max_abs_row_sum_norm(tf.matmul(Q, Q, transpose_a=True) - eye)/p, eps)
b = lambda Q: ortho_step(Q)
return tf.while_loop(c, b, [W], back_prop=back_prop)
def decode_for_training(cell, final_enc_state, labels):
# [actual batch size, max seq len, decoder cell size]
tf.assert_rank(labels, 3)
cell_size = cell.output_size
context = bridge(final_enc_state, cell_size)
# [actual batch size, decoder cell size]
assert context.get_shape().as_list() == [None, cell_size]
# tf.shape(labels): tuple of 1 element
batch_size = tf.shape(labels)[0] # type: tf.Tensor of rank 0
max_time_step = labels.get_shape()[1].value
with tf.variable_scope('decoder'):
def cond(loop_cnt, _, __, ___):
return tf.less(loop_cnt, max_time_step)
def body(loop_cnt, prev_label, prev_state, losses):
cell_output, state = cell(prev_label, prev_state)
output = decoder_projection(cell_output, cell_size)
# cut out the `loop_cnt`-th label
label = tf.reshape(
tf.slice(labels, begin=[0, loop_cnt, 0], size=[batch_size, 1, cell_size]),
shape=[batch_size, cell_size]
)
# loss for output past the last time step is calculated to be 0
loss = tf.nn.softmax_cross_entropy_with_logits(
logits=output,
labels=label
)
return (
tf.add(loop_cnt, 1),
# pass the label as the output of the current step
label,
state,
losses.write(loop_cnt, loss)
)
_, _, _, result_loss = tf.while_loop(
cond,
body,
loop_vars=(
tf.constant(0),
context,
cell.zero_state(batch_size=batch_size, dtype=tf.float32),
tf.TensorArray(tf.float32, size=0, dynamic_size=True)
),
)
losses = tf.reduce_sum(result_loss.stack(), axis=0)
time_steps = tf.reduce_sum(tf.reduce_sum(labels, axis=2), axis=1)
return tf.div(losses, time_steps)
def remap_keys(sparse_tensor):
# Current indices of our SparseTensor that we need to fix
bad_indices = sparse_tensor.indices
# Current values of our SparseTensor that we need to fix
bad_values = sparse_tensor.values
# Group by the batch_indices and get the count for each
size = tf.segment_sum(data = tf.ones_like(bad_indices[:,0], dtype = tf.int64), segment_ids = bad_indices[:,0]) - 1
# The number of batch_indices (this should be batch_size unless it is a partially full batch)
length = tf.shape(size, out_type = tf.int64)[0]
# Finds the cumulative sum which we can use for indexing later
cum = tf.cumsum(size)
# The offsets between each example in the batch due to our concatentation of the keys in the decode_example method
length_range = tf.range(start = 0, limit = length, delta = 1, dtype = tf.int64)
# Indices of the SparseTensor's indices member of the rows we added by the concatentation of our keys in the decode_example method
cum_range = cum + length_range
# The keys that we have extracted back out of our concatentated SparseTensor
gathered_indices = tf.squeeze(tf.gather(bad_indices, cum_range)[:,1])
# The enumerated row indices of the SparseTensor's indices member
sparse_indices_range = tf.range(tf.shape(bad_indices, out_type = tf.int64)[0], dtype = tf.int64)
# We want to find here the row indices of the SparseTensor's indices member that are of our actual data and not the concatentated rows
# So we want to find the intersection of the two sets and then take the opposite of that
x = sparse_indices_range
s = cum_range
# Number of multiples we are going to tile x, which is our sparse_indices_range
tile_multiples = tf.concat([tf.ones(tf.shape(tf.shape(x)), dtype=tf.int64), tf.shape(s, out_type = tf.int64)], axis = 0)
# Expands x, our sparse_indices_range, into a rank 2 tensor and then multiplies the rows by 1 (no copying) and the columns by the number of examples in the batch
x_tile = tf.tile(tf.expand_dims(x, -1), tile_multiples)
# Essentially a vectorized logical or, that we then negate
x_not_in_s = ~tf.reduce_any(tf.equal(x_tile, s), -1)
# The SparseTensor's indices that are our actual data by using the boolean_mask we just made above applied to the entire indices member of our SparseTensor
selected_indices = tf.boolean_mask(tensor = bad_indices, mask = x_not_in_s, axis = 0)
# Apply the same boolean_mask to the entire values member of our SparseTensor to get the actual values data
selected_values = tf.boolean_mask(tensor = bad_values, mask = x_not_in_s, axis = 0)
# Need to replace the first column of our selected_indices with keys, so we first need to tile our gathered_indices
tiling = tf.tile(input = tf.expand_dims(gathered_indices[0], -1), multiples = tf.expand_dims(size[0] , -1))
# We have to repeatedly apply the tiling to each example in the batch
# Since it is jagged we cannot use tf.map_fn due to the stacking of the TensorArray, so we have to create our own custom version
def loop_body(i, tensor_grow):
return i + 1, tf.concat(values = [tensor_grow, tf.tile(input = tf.expand_dims(gathered_indices[i], -1), multiples = tf.expand_dims(size[i] , -1))], axis = 0)
_, result = tf.while_loop(lambda i, tensor_grow: i < length, loop_body, [tf.constant(1, dtype = tf.int64), tiling])
# Concatenate tiled keys with the 2nd column of selected_indices
selected_indices_fixed = tf.concat([tf.expand_dims(result, -1), tf.expand_dims(selected_indices[:, 1], -1)], axis = 1)
# Combine everything together back into a SparseTensor
remapped_sparse_tensor = tf.SparseTensor(indices = selected_indices_fixed, values = selected_values, dense_shape = sparse_tensor.dense_shape)
return remapped_sparse_tensor
def infer(encoder_cell, decoder_cell, sentences):
tf.assert_rank(sentences, 3)
assert sentences.get_shape()[0].value == 1 # batch size
assert sentences.get_shape()[2].value == FEATURE_SIZE
# stops generating output if the length reaches the double of the source
output_len_threshold = sentences.get_shape()[1].value * 2
final_state_tuple = encode(sentences, encoder_cell, reuse=True)
context = bridge(final_state_tuple.c, decoder_cell.output_size, reuse=True)
with tf.variable_scope('decoder', reuse=True):
def cond(loop_cnt, prev_out, _, __):
less = tf.less(loop_cnt, output_len_threshold)
is_regular_word = tf.reduce_any(
tf.not_equal(
prev_out,
tf.one_hot([0], FEATURE_SIZE) # <eos>
)
)
return tf.logical_and(less, is_regular_word)
def body(loop_cnt, prev_out, prev_state, result):
cell_output, state = decoder_cell(prev_out, prev_state)
num_outputs = decoder_cell.output_size
output = decoder_projection(
cell_output,
num_outputs=num_outputs,
reuse=True
)
arg_max = tf.arg_max(output, dimension=1)
one_hot_output = tf.one_hot(
indices=arg_max,
depth=num_outputs
)
return (
tf.add(loop_cnt, 1),
one_hot_output,
state,
result.write(result.size(), tf.cast(one_hot_output, dtype=tf.int8))
)
_, __, ___, inferred = tf.while_loop(
cond,
body,
loop_vars=(
tf.constant(0),
context,
decoder_cell.zero_state(batch_size=1, dtype=tf.float32),
tf.TensorArray(tf.int8, size=0, dynamic_size=True)
)
)
return inferred.stack()
def mat_pow(matrix, power):
_, r = tf.while_loop(
lambda i, _: i > 0,
lambda i, c: [i-1, tf.matmul(c, matrix, transpose_a=True)],
[power, matrix],
parallel_iterations=1,
back_prop=False,
swap_memory=False
)
return r
def _minimize_in_graph(build_loss_fn, num_steps=200, optimizer=None):
"""Run an optimizer within the graph to minimize a loss function."""
optimizer = tf.train.AdamOptimizer(0.1) if optimizer is None else optimizer
def train_loop_body(step):
train_op = optimizer.minimize(
build_loss_fn if tf.executing_eagerly() else build_loss_fn())
return tf.tuple([tf.add(step, 1)], control_inputs=[train_op])
return tf.while_loop(cond=lambda step: step < num_steps,
body=train_loop_body,
loop_vars=[tf.constant(0)])
def csoftmax_for_slice(input):
""" It is a implementation of the constrained softmax (csoftmax) for slice.
Based on the paper:
https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers" (page 4)
Args:
input: A list of [input tensor, cumulative attention].
Returns:
output: A list of [csoftmax results, masks]
"""
[ten, u] = input
shape_t = ten.shape
shape_u = u.shape
ten -= tf.reduce_mean(ten)
q = tf.exp(ten)
active = tf.ones_like(u, dtype=tf.int32)
mass = tf.constant(0, dtype=tf.float32)
found = tf.constant(True, dtype=tf.bool)
def loop(q_, mask, mass_, found_):
q_list = tf.dynamic_partition(q_, mask, 2)
condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[0]), mask, 2) # 0 element it False,
# 1 element if true
p = q_list[1] * (1.0 - mass_) / tf.reduce_sum(q_list[1])
p_new = tf.dynamic_stitch(condition_indices, [q_list[0], p])
# condition verification and mask modification
less_mask = tf.cast(tf.less(u, p_new), tf.int32) # 0 when u is bigger than p, 1 when u is less than p
condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[0]), less_mask,
2) # 0 when u is bigger than p, 1 when u is less than p
split_p_new = tf.dynamic_partition(p_new, less_mask, 2)
split_u = tf.dynamic_partition(u, less_mask, 2)
alpha = tf.dynamic_stitch(condition_indices, [split_p_new[0], split_u[1]])
mass_ += tf.reduce_sum(split_u[1])
mask = mask * (tf.ones_like(less_mask) - less_mask)
found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), 0),
lambda: False,
lambda: True)
alpha = tf.reshape(alpha, q_.shape)
return alpha, mask, mass_, found_
(csoft, mask_, _, _) = tf.while_loop(cond=lambda _0, _1, _2, f: f,
body=loop,
loop_vars=(q, active, mass, found))
return [csoft, mask_]
def sample_sequence(*, hparams, length, start_token=None, batch_size=None, context=None, temperature=1, top_k=0):
if start_token is None:
assert context is not None, 'Specify exactly one of start_token and context!'
else:
assert context is None, 'Specify exactly one of start_token and context!'
context = tf.fill([batch_size, 1], start_token)
def step(hparams, tokens, past=None):
lm_output = model.model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(model.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
with tf.name_scope('sample_sequence'):
# Don't feed the last context token -- leave that to the loop below
# TODO: Would be slightly faster if we called step on the entire context,
# rather than leaving the last token transformer calculation to the while loop.
context_output = step(hparams, context[:, :-1])
def body(past, prev, output):
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
logits = next_outputs['logits'][:, -1, :] / tf.to_float(temperature)
logits = top_k_logits(logits, k=top_k)
samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
return [
tf.concat([past, next_outputs['presents']], axis=-2),
tf.squeeze(samples, axis=[1]),
tf.concat([output, samples], axis=1),
]
def cond(*args):
return True
_, _, tokens = tf.while_loop(
cond=cond, body=body,
maximum_iterations=length,
loop_vars=[
context_output['presents'],
context[:, -1],
context,
],
shape_invariants=[
tf.TensorShape(model.past_shape(hparams=hparams, batch_size=batch_size)),
tf.TensorShape([batch_size]),
tf.TensorShape([batch_size, None]),
],
back_prop=False,
)
return tokens
def _compute_backwards_messages(self, xs):
"""Computes the backwards messages used in smoothing."""
batch_size = tf.shape(xs)[1]
num_xs = tf.shape(xs)[0]
until_t = self.num_timesteps - num_xs
xs = tf.TensorArray(dtype=xs.dtype,
size=num_xs,
dynamic_size=False,
clear_after_read=True).unstack(xs)
messages_ta = tf.TensorArray(dtype=xs.dtype,
size=num_xs,
dynamic_size=False,
clear_after_read=False)
def compute_message(t, prev_mean, prev_prec, messages_ta):
"""Computes one step of the backwards messages."""
z_var = self.transition_variances.read(t)
w_z = self.transition_weights.read(t-1)
x_var = self.emission_variances.read(t)
w_x = self.emission_weights.read(t)
cur_x = xs.read(t - until_t)
# If it isn't the first message, add the terms from the transition.
def transition_term():
return (tf.square(self.transition_weights.read(t))/
self.transition_variances.read(t+1))
unary_prec = 1/z_var + tf.square(w_x)/x_var
unary_prec += tf.cond(tf.less(t, self.num_timesteps-1),
transition_term, lambda: 0.)
unary_mean = (w_x / x_var) * cur_x
pairwise_prec = w_z / z_var
next_prec = -tf.square(pairwise_prec)/(unary_prec + prev_prec)
next_mean = (pairwise_prec * (unary_mean + prev_mean) /
(unary_prec + prev_prec))
next_prec = tf.reshape(next_prec, [batch_size])
next_mean = tf.reshape(next_mean, [batch_size])
messages_ta = messages_ta.write(t - until_t,
tf.stack([next_mean, next_prec]))
return t-1, next_mean, next_prec, messages_ta
def pred(t, *unused_args):
return tf.greater_equal(t, until_t)
init_prec = tf.zeros([batch_size], dtype=xs.dtype)
init_mean = tf.zeros([batch_size], dtype=xs.dtype)
t0 = tf.constant(self.num_timesteps - 1, dtype=tf.int32)
outs = tf.while_loop(pred, compute_message,
(t0, init_mean, init_prec, messages_ta))
messages = outs[-1]
return messages
def _scan_body(args_list, num_steps):
"""Closure which implements `tf.scan` body."""
current_state, previous_kernel_results = args_list
return tf.while_loop(
cond=lambda it_, *args: it_ < num_steps,
body=lambda it_, cs, pkr: [it_ + 1] + list(kernel.one_step(cs, pkr)),
loop_vars=[
np.int32(0), # it_
current_state,
previous_kernel_results,
],
parallel_iterations=parallel_iterations)[1:] # Lop off `it_`.
def graph(self, x_tensor):
nb_classes = self.model.output_dim
nb_features = self.model.input_dim
# Compute our initial search domain. We optimize the initial search domain
# by removing all features that are already at their maximum values (if
# increasing input features).
search_domain = tf.reshape(
tf.cast(x_tensor < self.clip_max_input, tf.float32),
[-1, nb_features])
# y_in_init = tf.reshape(tf.one_hot(self.targed_y_input, depth=nb_classes), [-1, nb_classes])
# Loop variables
# x_in: the tensor that holds the latest adversarial outputs that are in
# progress.
# y_in: the tensor for target labels
# domain_in: the tensor that holds the latest search domain
# cond_in: the boolean tensor to show if more iteration is needed for
# generating adversarial samples
def _cond(x_in, domain_in, i, cond_in):
# Repeat the loop until we have achieved misclassification or
# reaches the maximum iterations
return tf.logical_and(tf.less(i, self.iterations), cond_in)
def _body(x_in, domain_in, i, cond_in):
logits = self.model.get_logits(x_in)
preds = tf.nn.softmax(logits)
preds_onehot = tf.one_hot(tf.argmax(preds, axis=1),
depth=nb_classes)
# get corresponding derivatives
derivatives, = tf.gradients(tf.reduce_mean(
preds[:, 0]), x_in) # malicious samples are labeled as '1'
# Remove the already-used input features from the search space
# Subtract 2 times the maximum value from those value so that
# they won't be picked later
increase_coef = 2 * tf.cast(tf.equal(domain_in, 0), tf.float32)
derivatives -= increase_coef \
* tf.reduce_max(tf.abs(derivatives), axis=1, keepdims=True)
derivatives = tf.reshape(derivatives, shape=[-1, nb_features])
# Create a mask to only keep features that match conditions
scores_mask = derivatives > 0
# Extract the best malware feature
scores = tf.cast(scores_mask, tf.float32) * derivatives
best = tf.argmax(scores, axis=1)
p1_one_hot = tf.one_hot(best, depth=nb_features)
# Check if more modification is needed for each sample
# mod_not_done = tf.equal(tf.reduce_sum(y_in * preds_onehot, axis=1), 0)
mod_not_done = tf.equal(preds_onehot[:, 0], 0)
if self.force_iteration:
cond = (tf.reduce_sum(
domain_in * tf.cast(scores_mask, tf.float32), axis=1) >= 1)
else:
cond = mod_not_done & (tf.reduce_sum(
domain_in * tf.cast(scores_mask, tf.float32), axis=1) >= 1)
cond_float = tf.reshape(tf.cast(cond, tf.float32), shape=[-1, 1])
to_mod = p1_one_hot * cond_float
domain_out = domain_in - to_mod
to_mod_reshape = tf.reshape(to_mod,
shape=([-1] +
x_in.shape[1:].as_list()))
x_out = tf.minimum(x_in + to_mod_reshape, self.scaled_clip_max)
# Increase the iterator, and check if all miss-classifications are done
i_out = tf.add(i, 1)
cond_out = tf.reduce_any(cond)
return x_out, domain_out, i_out, cond_out
x_adv_batch, _2, _3, _4 = tf.while_loop(
_cond, _body, [x_tensor, search_domain, 0, True])
return x_adv_batch
def build_sampling_graph(self, config, tokenizer, max_length=12):
if self.samples is not None:
return
# define stopping conditions
def stop_cond(states_c, states_h, tokens, seq_length, stop_indicator):
has_unfinished_dialogue = tf.less(
tf.shape(tf.where(stop_indicator))[0],
tf.shape(stop_indicator)
[0]) # TODO use "any" instead of checking shape
has_not_reach_size_limit = tf.less(tf.reduce_max(seq_length),
max_length)
return tf.logical_and(has_unfinished_dialogue,
has_not_reach_size_limit)
# define one_step sampling
with tf.variable_scope(self.scope_name):
stop_token = tf.constant(tokenizer.stop_token)
stop_dialogue_token = tf.constant(tokenizer.stop_dialogue)
def step(prev_state_c, prev_state_h, tokens, seq_length,
stop_indicator):
input = tf.gather(tokens, tf.shape(tokens)[0] - 1)
# Look for new finish dialogue
is_stop_token = tf.equal(input, stop_token)
is_stop_dialogue_token = tf.equal(input, stop_dialogue_token)
is_stop = tf.logical_or(is_stop_token, is_stop_dialogue_token)
stop_indicator = tf.logical_or(
stop_indicator, is_stop) # flag to false new finished dialogue
# increment seq_length when the dialogue is not over
seq_length = tf.where(stop_indicator, seq_length,
tf.add(seq_length, 1))
# compute the next words. TODO: factorize with qgen.. but how?!
with tf.variable_scope(self.scope_name, reuse=True):
word_emb = utils.get_embedding(
input,
n_words=tokenizer.no_words,
n_dim=config['word_embedding_size'],
scope="word_embedding",
reuse=True)
inp_emb = tf.concat([word_emb, self.image_emb], axis=1)
with tf.variable_scope("word_decoder"):
lstm_cell = tf.contrib.rnn.LayerNormBasicLSTMCell(
config['num_lstm_units'],
layer_norm=False,
dropout_keep_prob=1.0,
reuse=True)
state = tf.contrib.rnn.LSTMStateTuple(c=prev_state_c,
h=prev_state_h)
out, state = lstm_cell(inp_emb, state)
# store/update the state when the dialogue is not finished (after sampling the <?> token)
cond = tf.greater_equal(
seq_length, tf.subtract(tf.reduce_max(seq_length), 1))
state_c = tf.where(cond, state.c, prev_state_c)
state_h = tf.where(cond, state.h, prev_state_h)
with tf.variable_scope('decoder_output'):
output = utils.fully_connected(state_h,
tokenizer.no_words,
reuse=True)
sampled_tokens = tf.cond(
self.greedy, lambda: tf.argmax(output, 1),
lambda: tf.reshape(tf.multinomial(output, 1), [-1]))
sampled_tokens = tf.cast(sampled_tokens, tf.int32)
tokens = tf.concat(
[tokens, tf.expand_dims(sampled_tokens, 0)],
axis=0) # check axis!
return state_c, state_h, tokens, seq_length, stop_indicator
# initialialize sequences
batch_size = tf.shape(self.seq_length)[0]
seq_length = tf.fill([batch_size], 0)
stop_indicator = tf.fill([batch_size], False)
transpose_dialogue = tf.transpose(self.dialogues, perm=[1, 0])
self.samples = tf.while_loop(
stop_cond,
step, [
self.decoder_zero_state_c, self.decoder_zero_state_h,
transpose_dialogue, seq_length, stop_indicator
],
shape_invariants=[
self.decoder_zero_state_c.get_shape(),
self.decoder_zero_state_h.get_shape(),
tf.TensorShape([None, None]),
seq_length.get_shape(),
stop_indicator.get_shape()
])
def __call__(self,
c_x,
n,
d,
k,
combination,
eps_passed,
sample,
scope=None):
'''
Args:
c_x - tensor to be filled up with random walk property
n - number of nodes
d - number of features in the feature matrix
k - length of random walk
defaults to be None
'''
with tf.variable_scope(scope or type(self).__name__):
'''
c_x = input_layer(c_x, self.adj, self.weight, self.features, k, n, d, activation=None, batch_norm=False, istrain=False, scope=None)
c_x = tf.Print(c_x,[c_x], message="my c_x-values:")
with tf.variable_scope("Prior"):
prior_mu = tf.zeros(shape=[n,self.z_dim,1],name="prior_mu")
prior_sigma = tf.matrix_diag(tf.ones(shape=[n,self.z_dim]),name="prior_sigma")
with tf.variable_scope("Encoder"):
list_cx = tf.unstack(c_x)
# output will be of shape n X kd
enc_hidden = fc_layer(tf.concat(list_cx,1), k*d, activation=tf.nn.relu, scope="hidden")
#output will be of shape n X 5 (this is a hyper paramater)
enc_mu = fc_layer(enc_hidden, self.z_dim,activation=tf.nn.softplus, scope='mu')
enc_mu = tf.reshape(enc_mu, [n,self.z_dim,1])
enc_mu = tf.Print(enc_mu,[enc_mu], message="my enc_mu-values:")
# output will be n X 1 then convert that to a diagonal matrix
debug_sigma = fc_layer(enc_hidden, self.z_dim, activation=tf.nn.softplus, scope='sigma')
debug_sigma = tf.Print(debug_sigma,[debug_sigma], message="my debug_sigma-values:")
enc_sigma = tf.matrix_diag(debug_sigma, name="enc_sigma")
enc_sigma = tf.Print(enc_sigma,[enc_sigma], message="my enc_sigma-values:")
# Random sampling ~ N(0, 1)
eps = eps_passed
temp_stack = []
for i in range(n):
temp_stack.append(tf.matmul(enc_sigma[i], eps[i]))
z = tf.add(enc_mu, tf.stack(temp_stack))
# While we are trying to sample some edges, we sample Z from prior
'''
eps = eps_passed
#if sample:
z = eps
z = tf.Print(z, [z], message="my z-values:")
with tf.variable_scope("Poisson"):
#nodes = tf.reshape(self.node_count, [-1, self.z_dim])
#lambda_node = fc_layer(nodes, 1, activation=tf.nn.softplus, scope = "node")
lambda_node = fc_layer(tf.transpose(self.node_count),
1,
activation=tf.nn.softplus,
scope="node")
z_reshape = tf.reshape(z, [-1, self.z_dim])
z_reshape = tf.Print(z_reshape, [z_reshape],
message="my z-reshape-values:")
n_cast = tf.fill([1, self.z_dim], tf.cast(n, dtype=tf.float32))
z_concat = tf.concat(values=(z_reshape, n_cast), axis=0)
print("Debug concat z", z_concat.get_shape())
lambda_edge = fc_layer(z_concat,
1,
activation=tf.nn.softplus,
scope="edge")
def loop_cond(t, k, z, z_stack, z_stack_weight):
N = tf.stack([tf.shape(t)[0]])[0]
#N= tf.Print(N, [N], message="my N-values")
return tf.less(k, N)
def body(t, k, z, z_stack, z_stack_weight):
# need to check once sanity
print("Debug shape", t[k].get_shape(), tf.shape(z))
print("Debug shape gather",
tf.gather(z, t[k][0]).get_shape(), z.get_shape())
dots = tf.concat(values=([tf.gather(z, t[k][0])],
[tf.gather(z, t[k][1])]),
axis=1)
print("Debug shape dots", dots.get_shape())
for j in range(self.bin_dim):
m = np.zeros((1, self.bin_dim))
m[0][j] = 1
temp = tf.concat(values=(dots, tf.cast(m, tf.float32)),
axis=1)
z_stack_weight = tf.concat(values=(z_stack_weight, temp),
axis=0)
return (t, k + 1, z, tf.concat(values=(z_stack, dots),
axis=0), z_stack_weight)
k = tf.constant(0)
z_new = tf.reshape(z, [n, self.z_dim])
print("Debug z shape", z_new.get_shape())
dec_hidden = []
weight = []
#label = []
with tf.variable_scope("Decoder", reuse=tf.AUTO_REUSE):
z_stack_label = []
for u in range(n):
for j in range(4):
# we considered 4 types of atom C, H, O, N
m = np.zeros((1, 4))
m[0][j] = 1
z_stack_label.append(
tf.concat(values=(tf.transpose(z[u]), m),
axis=1)[0])
label = fc_layer(tf.stack(z_stack_label),
1,
activation=tf.nn.softplus,
scope="label")
for i in range(combination):
z_stack = tf.constant(0,
shape=[1, 2 * self.z_dim],
dtype=tf.float32)
z_stack_weight = tf.constant(
0,
shape=[1, 2 * self.z_dim + self.bin_dim],
dtype=tf.float32)
t = self.edges[i]
_, _, _, z_stack, z_stack_weight = tf.while_loop(
loop_cond,
body, [t, k, z_new, z_stack, z_stack_weight],
shape_invariants=[
t.get_shape(),
k.get_shape(),
z_new.get_shape(),
tf.TensorShape([None, 2 * self.z_dim]),
tf.TensorShape([None, 2 * self.z_dim + self.bin_dim])
])
with tf.variable_scope("Decoder", reuse=tf.AUTO_REUSE):
dec_hidden.append(
fc_layer(z_stack[1:],
1,
activation=tf.nn.softplus,
scope="hidden"))
weight.append(
fc_layer(z_stack_weight[1:],
1,
activation=tf.nn.softplus,
scope="marker"))
#label.append(fc_layer(tf.stack(z_stack_label), 1, activation=tf.nn.relu, scope = "label"))
return (c_x, dec_hidden, z, weight, label, tf.reduce_mean(lambda_node),
tf.reduce_mean(lambda_edge))
def construct_beam_search_functions(models, beam_size):
"""
Strategy:
compute the log_probs - same as with sampling
for sentences that are ended set log_prob(<eos>)=0, log_prob(not eos)=-inf
add previous cost to log_probs
run top k -> (idxs, values)
use values as new costs
divide idxs by num_classes to get state_idxs
use gather to get new states
take the remainder of idxs after num_classes to get new_predicted words
"""
# Get some parameter settings. For ensembling, some parameters are required
# to be consistent across all models but others are not. In the former
# case, we assume that consistency has already been checked. For the
# parameters that are allowed to vary across models, the first model's
# settings take precedence.
decoder = models[0].decoder
batch_size = tf.shape(decoder.init_state)[0]
embedding_size = decoder.embedding_size
translation_maxlen = decoder.translation_maxlen
target_vocab_size = decoder.target_vocab_size
high_depth = 0 if decoder.high_gru_stack == None \
else len(decoder.high_gru_stack.grus)
# Initialize loop variables
i = tf.constant(0)
init_ys = -tf.ones(dtype=tf.int32, shape=[batch_size])
init_embs = [tf.zeros(dtype=tf.float32, shape=[batch_size,embedding_size])] * len(models)
f_min = numpy.finfo(numpy.float32).min
init_cost = [0.] + [f_min]*(beam_size-1) # to force first top k are from first hypo only
init_cost = tf.constant(init_cost, dtype=tf.float32)
init_cost = tf.tile(init_cost, multiples=[batch_size/beam_size])
ys_array = tf.TensorArray(
dtype=tf.int32,
size=translation_maxlen,
clear_after_read=True,
name='y_sampled_array')
p_array = tf.TensorArray(
dtype=tf.int32,
size=translation_maxlen,
clear_after_read=True,
name='parent_idx_array')
init_base_states = [m.decoder.init_state for m in models]
init_high_states = [[m.decoder.init_state] * high_depth for m in models]
init_loop_vars = [i, init_base_states, init_high_states, init_ys, init_embs,
init_cost, ys_array, p_array]
# Prepare cost matrix for completed sentences -> Prob(EOS) = 1 and Prob(x) = 0
eos_log_probs = tf.constant(
[[0.] + ([f_min]*(target_vocab_size - 1))],
dtype=tf.float32)
eos_log_probs = tf.tile(eos_log_probs, multiples=[batch_size,1])
def cond(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
return tf.logical_and(
tf.less(i, translation_maxlen),
tf.reduce_any(tf.not_equal(prev_ys, 0)))
def body(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
# get predictions from all models and sum the log probs
sum_log_probs = None
base_states = [None] * len(models)
high_states = [None] * len(models)
for j in range(len(models)):
d = models[j].decoder
states1 = d.grustep1.forward(prev_base_states[j], prev_embs[j])
att_ctx = d.attstep.forward(states1)
base_states[j] = d.grustep2.forward(states1, att_ctx)
if d.high_gru_stack == None:
stack_output = base_states[j]
high_states[j] = []
else:
if d.high_gru_stack.context_state_size == 0:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j])
else:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j], context=att_ctx)
logits = d.predictor.get_logits(prev_embs[j], stack_output,
att_ctx, multi_step=False)
log_probs = tf.nn.log_softmax(logits) # shape (batch, vocab_size)
if sum_log_probs == None:
sum_log_probs = log_probs
else:
sum_log_probs += log_probs
# set cost of EOS to zero for completed sentences so that they are in top k
# Need to make sure only EOS is selected because a completed sentence might
# kill ongoing sentences
sum_log_probs = tf.where(tf.equal(prev_ys, 0), eos_log_probs, sum_log_probs)
all_costs = sum_log_probs + tf.expand_dims(cost, axis=1) # TODO: you might be getting NaNs here since -inf is in log_probs
all_costs = tf.reshape(all_costs,
shape=[-1, target_vocab_size * beam_size])
values, indices = tf.nn.top_k(all_costs, k=beam_size) #the sorted option is by default True, is this needed?
new_cost = tf.reshape(values, shape=[batch_size])
offsets = tf.range(
start = 0,
delta = beam_size,
limit = batch_size,
dtype=tf.int32)
offsets = tf.expand_dims(offsets, axis=1)
survivor_idxs = (indices/target_vocab_size) + offsets
new_ys = indices % target_vocab_size
survivor_idxs = tf.reshape(survivor_idxs, shape=[batch_size])
new_ys = tf.reshape(new_ys, shape=[batch_size])
new_embs = [m.decoder.y_emb_layer.forward(new_ys, factor=0) for m in models]
new_base_states = [tf.gather(s, indices=survivor_idxs) for s in base_states]
new_high_states = [[tf.gather(s, indices=survivor_idxs) for s in states] for states in high_states]
new_cost = tf.where(tf.equal(new_ys, 0), tf.abs(new_cost), new_cost)
ys_array = ys_array.write(i, value=new_ys)
p_array = p_array.write(i, value=survivor_idxs)
return i+1, new_base_states, new_high_states, new_ys, new_embs, new_cost, ys_array, p_array
final_loop_vars = tf.while_loop(
cond=cond,
body=body,
loop_vars=init_loop_vars,
back_prop=False)
i, _, _, _, _, cost, ys_array, p_array = final_loop_vars
indices = tf.range(0, i)
sampled_ys = ys_array.gather(indices)
parents = p_array.gather(indices)
cost = tf.abs(cost) #to get negative-log-likelihood
return sampled_ys, parents, cost
def run_episode():
time_step = tf_env.reset()
return tf.while_loop(cond=c, body=body, loop_vars=[time_step])
def homography_adaptation(image, net, config):
"""Perfoms homography adaptation.
Inference using multiple random warped patches of the same input image for robust
predictions.
Arguments:
image: A `Tensor` with shape `[N, H, W, 1]`.
net: A function that takes an image as input, performs inference, and outputs the
prediction dictionary.
config: A configuration dictionary containing optional entries such as the number
of sampled homographies `'num'`, the aggregation method `'aggregation'`.
Returns:
A dictionary which contains the aggregated detection probabilities.
"""
probs = net(image)['prob']
counts = tf.ones_like(probs)
images = image
probs = tf.expand_dims(probs, axis=-1)
counts = tf.expand_dims(counts, axis=-1)
images = tf.expand_dims(images, axis=-1)
shape = tf.shape(image)[1:3]
config = dict_update(homography_adaptation_default_config, config)
def step(i, probs, counts, images):
# Sample image patch
H = sample_homography(shape, **config['homographies'])
H_inv = invert_homography(H)
warped = H_transform(image, H, interpolation='BILINEAR')
count = H_transform(tf.expand_dims(tf.ones(tf.shape(image)[:3]), -1),
H_inv, interpolation='NEAREST')[..., 0]
# Predict detection probabilities
warped_shape = tf.to_int32(
tf.to_float(shape)*config['homographies']['patch_ratio'])
input_warped = tf.image.resize_images(warped, warped_shape)
prob = net(input_warped)['prob']
prob = tf.image.resize_images(tf.expand_dims(prob, axis=-1), shape)[..., 0]
prob_proj = H_transform(tf.expand_dims(prob, -1), H_inv,
interpolation='BILINEAR')[..., 0]
probs = tf.concat([probs, tf.expand_dims(prob_proj, -1)], axis=-1)
counts = tf.concat([counts, tf.expand_dims(count, -1)], axis=-1)
images = tf.concat([images, tf.expand_dims(warped, -1)], axis=-1)
return i + 1, probs, counts, images
_, probs, counts, images = tf.while_loop(
lambda i, p, c, im: tf.less(i, config['num'] - 1),
step,
[0, probs, counts, images],
parallel_iterations=1,
back_prop=False,
shape_invariants=[
tf.TensorShape([]),
tf.TensorShape([None, None, None, None]),
tf.TensorShape([None, None, None, None]),
tf.TensorShape([None, None, None, 1, None])])
counts = tf.reduce_sum(counts, axis=-1)
max_prob = tf.reduce_max(probs, axis=-1)
mean_prob = tf.reduce_sum(probs, axis=-1) / counts
if config['aggregation'] == 'max':
prob = max_prob
elif config['aggregation'] == 'sum':
prob = mean_prob
else:
raise ValueError('Unkown aggregation method: {}'.format(config['aggregation']))
if config['filter_counts']:
prob = tf.where(tf.greater_equal(counts, config['filter_counts']),
prob, tf.zeros_like(prob))
return {'prob': prob, 'counts': counts,
'mean_prob': mean_prob, 'input_images': images, 'H_probs': probs} # debug
def _sample_n(self, n, seed=None):
seed = seed_stream.SeedStream(seed, salt='vom_mises_fisher')
# The sampling strategy relies on the fact that vMF variates are symmetric
# about the mean direction. Accordingly, if we have a sampling strategy for
# the away-from-mean angle, then we can uniformly sample the remaining
# dimensions on the S^{dim-2} sphere for , and rotate these samples from a
# (1, 0, 0, ..., 0)-mode distribution into the target orientation.
#
# This is easy to imagine on the 1-sphere (S^1; in 2-D space): sample a
# von-Mises distributed `x` value in [-1, 1], then uniformly select what
# amounts to a "up" or "down" additional degree of freedom after unit
# normalizing, followed by a final rotation to the desired mean direction
# from a basis of (1, 0).
#
# On S^2 (in 3-D), selecting a vMF `x` identifies a circle in `yz` on the
# unit sphere over which the distribution is uniform, in particular the
# circle where x = \hat{x} intersects the unit sphere. We pick a point on
# that circle, then rotate to the desired mean direction from a basis of
# (1, 0, 0).
event_dim = (tf.compat.dimension_value(self.event_shape[0])
or self._event_shape_tensor()[0])
sample_batch_shape = tf.concat([[n], self._batch_shape_tensor()],
axis=0)
dim = tf.cast(event_dim - 1, self.dtype)
if event_dim == 3:
samples_dim0 = self._sample_3d(n, seed=seed)
else:
# Wood'94 provides a rejection algorithm to sample the x coordinate.
# Wood'94 definition of b:
# b = (-2 * kappa + tf.sqrt(4 * kappa**2 + dim**2)) / dim
# https://stats.stackexchange.com/questions/156729 suggests:
b = dim / (2 * self.concentration +
tf.sqrt(4 * self.concentration**2 + dim**2))
# TODO(bjp): Integrate any useful numerical tricks from hyperspherical VAE
# https://github.com/nicola-decao/s-vae-tf/
x = (1 - b) / (1 + b)
c = self.concentration * x + dim * tf.math.log1p(-x**2)
beta = beta_lib.Beta(dim / 2, dim / 2)
def cond_fn(w, should_continue):
del w
return tf.reduce_any(input_tensor=should_continue)
def body_fn(w, should_continue):
z = beta.sample(sample_shape=sample_batch_shape, seed=seed())
w = tf.where(should_continue,
(1 - (1 + b) * z) / (1 - (1 - b) * z), w)
w = tf.debugging.check_numerics(w, 'w')
should_continue = tf.logical_and(
should_continue,
self.concentration * w + dim * tf.math.log1p(-x * w) - c <
tf.math.log(
tf.random.uniform(sample_batch_shape,
seed=seed(),
dtype=self.dtype)))
return w, should_continue
w = tf.zeros(sample_batch_shape, dtype=self.dtype)
should_continue = tf.ones(sample_batch_shape, dtype=tf.bool)
samples_dim0 = tf.while_loop(cond=cond_fn,
body=body_fn,
loop_vars=(w, should_continue))[0]
samples_dim0 = samples_dim0[..., tf.newaxis]
if not self._allow_nan_stats:
# Verify samples are w/in -1, 1, with useful error output tensors (top
# value rather than all values).
with tf.control_dependencies([
assert_util.assert_less_equal(
samples_dim0,
self.dtype.as_numpy_dtype(1.01),
data=[tf.nn.top_k(tf.reshape(samples_dim0, [-1]))[0]]),
assert_util.assert_greater_equal(
samples_dim0,
self.dtype.as_numpy_dtype(-1.01),
data=[
-tf.nn.top_k(tf.reshape(-samples_dim0, [-1]))[0]
])
]):
samples_dim0 = tf.identity(samples_dim0)
samples_otherdims_shape = tf.concat(
[sample_batch_shape, [event_dim - 1]], axis=0)
unit_otherdims = tf.nn.l2_normalize(tf.random.normal(
samples_otherdims_shape, seed=seed(), dtype=self.dtype),
axis=-1)
samples = tf.concat(
[
samples_dim0, # we must avoid sqrt(1 - (>1)**2)
tf.sqrt(tf.maximum(1 - samples_dim0**2, 0.)) * unit_otherdims
],
axis=-1)
samples = tf.nn.l2_normalize(samples, axis=-1)
if not self._allow_nan_stats:
samples = tf.debugging.check_numerics(samples, 'samples')
# Runtime assert that samples are unit length.
if not self._allow_nan_stats:
worst, idx = tf.nn.top_k(
tf.reshape(tf.abs(1 - tf.linalg.norm(tensor=samples, axis=-1)),
[-1]))
with tf.control_dependencies([
assert_util.assert_near(
self.dtype.as_numpy_dtype(0),
worst,
data=[
worst, idx,
tf.gather(tf.reshape(samples, [-1, event_dim]),
idx)
],
atol=1e-4,
summarize=100)
]):
samples = tf.identity(samples)
# The samples generated are symmetric around a mode at (1, 0, 0, ...., 0).
# Now, we move the mode to `self.mean_direction` using a rotation matrix.
if not self._allow_nan_stats:
# Assert that the basis vector rotates to the mean direction, as expected.
basis = tf.cast(
tf.concat([[1.], tf.zeros([event_dim - 1])], axis=0),
self.dtype)
with tf.control_dependencies([
assert_util.assert_less(
tf.linalg.norm(
tensor=self._rotate(basis) - self.mean_direction,
axis=-1), self.dtype.as_numpy_dtype(1e-5))
]):
return self._rotate(samples)
return self._rotate(samples)
def __batch_while_loop(
self, truth_padded_data: padder.PlaceholderPaddedData,
initial_hidden_vector_input: tf.Tensor,
num_batches: int) -> (tf.TensorArray, tf.TensorArray):
def body(step, batch_states_ta: tf.TensorArray,
batch_outputs_ta: tf.TensorArray,
batch_outputs_counts_ta: tf.TensorArray,
batch_step_counts_ta: tf.TensorArray):
with tf.variable_scope("initial_hidden_vector"):
current_initial_hidden_vector_input = tf.gather(
initial_hidden_vector_input,
step,
name="current_initial_hidden_vector_input")
current_hidden_vector = self.__create_fully_connected_layers(
current_initial_hidden_vector_input,
[self.hidden_vector_size])
with tf.variable_scope("step_while_loop"):
current_step_count = tf.gather(truth_padded_data.step_counts,
step,
name="current_step_count")
current_outputs_padded = tf.gather(
truth_padded_data.outputs_padded,
step,
name="current_outputs_padded")
current_outputs_counts = tf.gather(
truth_padded_data.outputs_counts,
step,
name="current_outputs_counts")
current_states, current_outputs, current_outputs_counts, current_step_count = \
self.__step_while_loop(
current_step_count,
current_outputs_padded,
current_outputs_counts,
current_hidden_vector)
return \
step + 1, \
batch_states_ta.write(step, current_states, "write_batch_states"), \
batch_outputs_ta.write(step, current_outputs, "write_batch_outputs"), \
batch_outputs_counts_ta.write(step, current_outputs_counts, "write_batch_outputs_counts"), \
batch_step_counts_ta.write(step, current_step_count, "write_step_counts")
def cond(step, *_):
return step < num_batches
*_, \
generated_states_padded_ta, \
generated_outputs_padded_ta, \
generated_outputs_counts_padded_ta, \
generated_step_counts_ta = tf.while_loop(
cond=cond,
body=body,
loop_vars=[
tf.constant(0),
tf.TensorArray(dtype=tf.float32, size=num_batches, name="batch_states_ta"),
tf.TensorArray(dtype=tf.float32, size=num_batches, name="batch_outputs_ta"),
tf.TensorArray(dtype=tf.int32, size=num_batches, name="batch_outputs_counts_ta"),
tf.TensorArray(dtype=tf.int32, size=num_batches, name="batch_step_counts_ta")])
return \
generated_states_padded_ta, \
generated_outputs_padded_ta, \
generated_outputs_counts_padded_ta, \
generated_step_counts_ta
size), facts_0s)
# Inverse attention mask, for what's retained in the state.
retain = 1 - attend_to
# GRU pass over the facts, according to the attention mask.
while_valid_index = (lambda state, index: index < tf.shape(cs)[1])
update_state = (
lambda state, index: (attend_to[:, index, :] * attention_gru(
cs[:, index, :], state)[0] + retain[:, index, :] * state))
# start loop with most recent memory and at the first index
memory.append(
tuple(
tf.while_loop(while_valid_index,
(lambda state, index:
(update_state(state, index), index + 1)),
loop_vars=[memory[-1], 0]))[0])
attends.append(attend_to)
# Reuse variables so the GRU pass uses the same variables every pass.
scope.reuse_variables()
# Answer Module
# a0: Final memory state. (Input to answer module)
a0 = tf.concat([memory[-1], q], -1)
# fc_init: Initializer for the final fully connected layer's weights.
fc_init = tf.random_normal_initializer(stddev=0.1)
def ExtractKeypointDescriptor(image, layer_name, image_scales, iou,
max_feature_num, abs_thres, model_fn):
"""Extract keypoint descriptor for input image.
Args:
image: A image tensor with shape [h, w, channels].
layer_name: The endpoint of feature extraction layer.
image_scales: A 1D float tensor which contains the scales.
iou: A float scalar denoting the IOU threshold for NMS.
max_feature_num: An int tensor denoting the maximum selected feature points.
abs_thres: A float tensor denoting the score threshold for feature
selection.
model_fn: Model function. Follows the signature:
* Args:
* `images`: Image tensor which is re-scaled.
* `normalized_image`: Whether or not the images are normalized.
* `reuse`: Whether or not the layer and its variables should be reused.
* Returns:
* `attention`: Attention score after the non-linearity.
* `feature_map`: Feature map obtained from the ResNet model.
Returns:
boxes: [N, 4] float tensor which denotes the selected receptive box. N is
the number of final feature points which pass through keypoint selection
and NMS steps.
feature_scales: [N] float tensor. It is the inverse of the input image
scales such that larger image scales correspond to larger image regions,
which is compatible with scale-space keypoint detection convention.
features: [N, depth] float tensor with feature descriptors.
scores: [N, 1] float tensor denoting the attention score.
Raises:
ValueError: If the layer_name is unsupported.
"""
original_image_shape_float = tf.gather(tf.to_float(tf.shape(image)),
[0, 1])
image_tensor = NormalizePixelValues(image)
image_tensor = tf.expand_dims(image_tensor, 0, name='image/expand_dims')
# Feature depth and receptive field parameters for each network version.
if layer_name == 'resnet_v1_50/block3':
feature_depth = 1024
rf, stride, padding = [291.0, 32.0, 145.0]
elif layer_name == 'resnet_v1_50/block4':
feature_depth = 2048
rf, stride, padding = [483.0, 32.0, 241.0]
else:
raise ValueError('Unsupported layer_name.')
def _ProcessSingleScale(scale_index,
boxes,
features,
scales,
scores,
reuse=True):
"""Resize the image and run feature extraction and keypoint selection.
This function will be passed into tf.while_loop() and be called
repeatedly. The input boxes are collected from the previous iteration
[0: scale_index -1]. We get the current scale by
image_scales[scale_index], and run image resizing, feature extraction and
keypoint selection. Then we will get a new set of selected_boxes for
current scale. In the end, we concat the previous boxes with current
selected_boxes as the output.
Args:
scale_index: A valid index in the image_scales.
boxes: Box tensor with the shape of [N, 4].
features: Feature tensor with the shape of [N, depth].
scales: Scale tensor with the shape of [N].
scores: Attention score tensor with the shape of [N].
reuse: Whether or not the layer and its variables should be reused.
Returns:
scale_index: The next scale index for processing.
boxes: Concatenated box tensor with the shape of [K, 4]. K >= N.
features: Concatenated feature tensor with the shape of [K, depth].
scales: Concatenated scale tensor with the shape of [K].
scores: Concatenated attention score tensor with the shape of [K].
"""
scale = tf.gather(image_scales, scale_index)
new_image_size = tf.to_int32(
tf.round(original_image_shape_float * scale))
resized_image = tf.image.resize_bilinear(image_tensor, new_image_size)
attention, feature_map = model_fn(resized_image,
normalized_image=True,
reuse=reuse)
rf_boxes = CalculateReceptiveBoxes(
tf.shape(feature_map)[1],
tf.shape(feature_map)[2], rf, stride, padding)
# Re-project back to the original image space.
rf_boxes = tf.divide(rf_boxes, scale)
attention = tf.reshape(attention, [-1])
feature_map = tf.reshape(feature_map, [-1, feature_depth])
# Use attention score to select feature vectors.
indices = tf.reshape(tf.where(attention >= abs_thres), [-1])
selected_boxes = tf.gather(rf_boxes, indices)
selected_features = tf.gather(feature_map, indices)
selected_scores = tf.gather(attention, indices)
selected_scales = tf.ones_like(selected_scores, tf.float32) / scale
# Concat with the previous result from different scales.
boxes = tf.concat([boxes, selected_boxes], 0)
features = tf.concat([features, selected_features], 0)
scales = tf.concat([scales, selected_scales], 0)
scores = tf.concat([scores, selected_scores], 0)
return scale_index + 1, boxes, features, scales, scores
output_boxes = tf.zeros([0, 4], dtype=tf.float32)
output_features = tf.zeros([0, feature_depth], dtype=tf.float32)
output_scales = tf.zeros([0], dtype=tf.float32)
output_scores = tf.zeros([0], dtype=tf.float32)
# Process the first scale separately, the following scales will reuse the
# graph variables.
(_, output_boxes, output_features, output_scales,
output_scores) = _ProcessSingleScale(0,
output_boxes,
output_features,
output_scales,
output_scores,
reuse=False)
i = tf.constant(1, dtype=tf.int32)
num_scales = tf.shape(image_scales)[0]
keep_going = lambda j, boxes, features, scales, scores: tf.less(
j, num_scales)
(_, output_boxes, output_features, output_scales,
output_scores) = tf.while_loop(cond=keep_going,
body=_ProcessSingleScale,
loop_vars=[
i, output_boxes, output_features,
output_scales, output_scores
],
shape_invariants=[
i.get_shape(),
tf.TensorShape([None, 4]),
tf.TensorShape([None, feature_depth]),
tf.TensorShape([None]),
tf.TensorShape([None])
],
back_prop=False)
feature_boxes = box_list.BoxList(output_boxes)
feature_boxes.add_field('features', output_features)
feature_boxes.add_field('scales', output_scales)
feature_boxes.add_field('scores', output_scores)
nms_max_boxes = tf.minimum(max_feature_num, feature_boxes.num_boxes())
final_boxes = box_list_ops.non_max_suppression(feature_boxes, iou,
nms_max_boxes)
return (final_boxes.get(), final_boxes.get_field('scales'),
final_boxes.get_field('features'),
tf.expand_dims(final_boxes.get_field('scores'), 1))
def inference_winner_take_all(images,
cams,
depth_num,
depth_start,
depth_end,
is_master_gpu=True,
reg_type='GRU',
inverse_depth=False):
""" infer disparity image from stereo images and cameras """
if not inverse_depth:
depth_interval = (depth_end -
depth_start) / (tf.cast(depth_num, tf.float32) - 1)
# reference image
ref_image = tf.squeeze(tf.slice(images, [0, 0, 0, 0, 0],
[-1, 1, -1, -1, 3]),
axis=1)
ref_cam = tf.squeeze(tf.slice(cams, [0, 0, 0, 0, 0], [-1, 1, 2, 4, 4]),
axis=1)
# image feature extraction
if is_master_gpu:
ref_tower = UNetDS2GN({'data': ref_image},
is_training=True,
reuse=False)
else:
ref_tower = UNetDS2GN({'data': ref_image},
is_training=True,
reuse=True)
view_towers = []
for view in range(1, FLAGS.view_num):
view_image = tf.squeeze(tf.slice(images, [0, view, 0, 0, 0],
[-1, 1, -1, -1, -1]),
axis=1)
view_tower = UNetDS2GN({'data': view_image},
is_training=True,
reuse=True)
view_towers.append(view_tower)
# get all homographies
view_homographies = []
for view in range(1, FLAGS.view_num):
view_cam = tf.squeeze(tf.slice(cams, [0, view, 0, 0, 0],
[-1, 1, 2, 4, 4]),
axis=1)
if inverse_depth:
homographies = get_homographies_inv_depth(ref_cam,
view_cam,
depth_num=depth_num,
depth_start=depth_start,
depth_end=depth_end)
else:
homographies = get_homographies(ref_cam,
view_cam,
depth_num=depth_num,
depth_start=depth_start,
depth_interval=depth_interval)
view_homographies.append(homographies)
# gru unit
gru1_filters = 16
gru2_filters = 4
gru3_filters = 2
feature_shape = [FLAGS.batch_size, FLAGS.max_h / 4, FLAGS.max_w / 4, 32]
gru_input_shape = [feature_shape[1], feature_shape[2]]
state1 = tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], gru1_filters])
state2 = tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], gru2_filters])
state3 = tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], gru3_filters])
conv_gru1 = ConvGRUCell(shape=gru_input_shape,
kernel=[3, 3],
filters=gru1_filters)
conv_gru2 = ConvGRUCell(shape=gru_input_shape,
kernel=[3, 3],
filters=gru2_filters)
conv_gru3 = ConvGRUCell(shape=gru_input_shape,
kernel=[3, 3],
filters=gru3_filters)
# initialize variables
exp_sum = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='exp_sum',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
depth_image = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='depth_image',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
max_prob_image = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='max_prob_image',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
init_map = tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1])
# define winner take all loop
def body(depth_index, state1, state2, state3, depth_image, max_prob_image,
exp_sum, incre):
"""Loop body."""
# calculate cost
ave_feature = ref_tower.get_output()
ave_feature2 = tf.square(ref_tower.get_output())
for view in range(0, FLAGS.view_num - 1):
homographies = view_homographies[view]
homographies = tf.transpose(homographies, perm=[1, 0, 2, 3])
homography = homographies[depth_index]
# warped_view_feature = homography_warping(view_towers[view].get_output(), homography)
warped_view_feature = tf_transform_homography(
view_towers[view].get_output(), homography)
ave_feature = ave_feature + warped_view_feature
ave_feature2 = ave_feature2 + tf.square(warped_view_feature)
ave_feature = ave_feature / FLAGS.view_num
ave_feature2 = ave_feature2 / FLAGS.view_num
cost = ave_feature2 - tf.square(ave_feature)
cost.set_shape(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 32])
# gru
reg_cost1, state1 = conv_gru1(-cost, state1, scope='conv_gru1')
reg_cost2, state2 = conv_gru2(reg_cost1, state2, scope='conv_gru2')
reg_cost3, state3 = conv_gru3(reg_cost2, state3, scope='conv_gru3')
reg_cost = tf.layers.conv2d(reg_cost3,
1,
3,
padding='same',
reuse=tf.AUTO_REUSE,
name='prob_conv')
prob = tf.exp(reg_cost)
# index
d_idx = tf.cast(depth_index, tf.float32)
if inverse_depth:
inv_depth_start = tf.div(1.0, depth_start)
inv_depth_end = tf.div(1.0, depth_end)
inv_interval = (inv_depth_start - inv_depth_end) / (
tf.cast(depth_num, 'float32') - 1)
inv_depth = inv_depth_start - d_idx * inv_interval
depth = tf.div(1.0, inv_depth)
else:
depth = depth_start + d_idx * depth_interval
temp_depth_image = tf.reshape(depth, [FLAGS.batch_size, 1, 1, 1])
temp_depth_image = tf.tile(
temp_depth_image,
[1, int(feature_shape[1]),
int(feature_shape[2]), 1
]) # modified by zhantao deng @ 26-07-2019, float -> int
# update the best
update_flag_image = tf.cast(tf.less(max_prob_image, prob),
dtype='float32')
new_max_prob_image = update_flag_image * prob + (
1 - update_flag_image) * max_prob_image
new_depth_image = update_flag_image * temp_depth_image + (
1 - update_flag_image) * depth_image
max_prob_image = tf.assign(max_prob_image, new_max_prob_image)
depth_image = tf.assign(depth_image, new_depth_image)
# update counter
exp_sum = tf.assign_add(exp_sum, prob)
depth_index = tf.add(depth_index, incre)
return depth_index, state1, state2, state3, depth_image, max_prob_image, exp_sum, incre
# run forward loop
exp_sum = tf.assign(exp_sum, init_map)
depth_image = tf.assign(depth_image, init_map)
max_prob_image = tf.assign(max_prob_image, init_map)
depth_index = tf.constant(0)
incre = tf.constant(1)
cond = lambda depth_index, *_: tf.less(depth_index, depth_num)
_, state1, state2, state3, depth_image, max_prob_image, exp_sum, incre = tf.while_loop(
cond,
body, [
depth_index, state1, state2, state3, depth_image, max_prob_image,
exp_sum, incre
],
back_prop=False,
parallel_iterations=1)
# get output
forward_exp_sum = exp_sum + 1e-7
forward_depth_map = depth_image
return forward_depth_map, max_prob_image / forward_exp_sum
def max_margin(labels, mentions, scores):
batch = 100
# current_mention = mentions[0]
# labels_to_process = []
# pred_to_process = []
# i_to_process = tf.Variable([], dtype=tf.int32, validate_shape=False, trainable=False, name='i_to_process')
# # i_to_process = []
# loss = []
# labels_to_process = tf.Variable([labels[0]], validate_shape=False, trainable=False)
# pred_to_process = tf.Variable([scores[0]], validate_shape=False, trainable=False)
def same(i):
# labels_to_process.append(label)
# pred_to_process.append(score)
# i_to_process.append(i)
# nonlocal i_to_process
#
# concat_i = tf.concat([i_to_process, [i]], 0, name='concat_i')
# assign_i = tf.assign(i_to_process, concat_i, validate_shape=False, name='assign_i')
# print_i = tf.Print(i_to_process, [i_to_process], 'i_to_process: ', first_n=20, summarize=100)
# i_to_process = tf.concat([i_to_process, i], 1)
# nonlocal labels_to_process
# nonlocal pred_to_process
#
# labels_to_process = tf.stack([labels_to_process, label])
# # assign_op_label = tf.assign(labels_to_process, concat_label, validate_shape=False)
#
# pred_to_process = tf.stack([pred_to_process, score])
# assign_op_score = tf.assign(pred_to_process, concat_score, validate_shape=False)
#
# with tf.control_dependencies([assign_op_label, assign_op_score]):
# labels_to_process = tf.Print(labels_to_process, data=[labels_to_process, labels_to_process.read_value()],
# message='labels_to_process, _read: ')
# labels_print = tf.Print(labels_to_process, data=[labels_to_process], message='labels_to_process: ')
# mention_print = tf.Print(mention, data=[mention], message='mention: ')
# with tf.control_dependencies([assign_i, print_i]):
return K.constant(0.)
def different(indices):
# nonlocal labels_to_process
# nonlocal pred_to_process
# nonlocal i_to_process
# labels = tf.stack(labels_to_process)
# labels_to_process = tf.Print(labels_to_process, [labels_to_process], 'labels_to_process:')
# predictions = tf.stack(pred_to_process)
# predictions = tf.Print(predictions, [predictions], 'predictions:')
labels_to_process = tf.gather(labels, indices)
predictions_to_process = tf.gather(scores, indices)
# labels_to_process = [label]
# pred_to_process = [score]
# i_to_process = [i]
# concat_i = tf.concat([[], [i]], 0)
# null_i = tf.Variable([], dtype=tf.int32, validate_shape=False, trainable=False, name='i_to_process_null')
# assign_i = tf.assign(i_to_process, null_i, validate_shape=False)
# print_i = tf.Print(i_to_process, [i_to_process], 'i_to_process: ', first_n=20, summarize=100)
# i_to_process = tf.concat([[], i], 1)
# labels_to_process = tf.concat([[], label], 0)
# labels_to_process = tf.Variable([label], validate_shape=False, trainable=False)
#
# # pred_to_process = tf.concat([[], score], 0)
# pred_to_process = tf.Variable([score], validate_shape=False, trainable=False)
# assign_op_label = tf.assign(labels_to_process, concat_label, validate_shape=False)
#
# concat_score = tf.stack([pred_to_process, score], 0)
# assign_op_score = tf.assign(pred_to_process, concat_score, validate_shape=False)
#
# with tf.control_dependencies([assign_op_label, assign_op_score]):
# labels_print = tf.Print(labels, data=[labels], message='labels: ')
# predictions_print = tf.Print(predictions, data=[predictions], message='predictions: ')
# with tf.control_dependencies([assign_i, print_i]):
return process_antecedents(labels_to_process, predictions_to_process)
labels_arr = tf.unstack(labels, num=batch)
# print('Labels_arr: {}'.format(labels_arr))
mentions_arr = tf.unstack(mentions, num=batch)
# print('Mentions_arr: {}'.format(mentions_arr))
scores_arr = tf.unstack(scores, num=batch)
# print('Scores_arr: {}'.format(scores_arr))
def mention_group_loss(i, prev_mention, indices):
condition = tf.equal(tf.gather(mentions, i), prev_mention)
return tf.cond(condition, lambda: K.constant(0.), lambda: different(indices))
def set_indices(i, prev_mention, indices):
condition = tf.equal(tf.gather(mentions, i), prev_mention)
return tf.cond(condition, lambda: tf.concat([indices, [i]], axis=0), lambda: tf.zeros([0], tf.int32))
i0 = tf.constant(0)
prev_mention0 = mentions_arr[0]
indices0 = tf.zeros([0], tf.int32)
loss0 = tf.constant(0.)
c = lambda i, prev_mention, indices, loss: tf.less(i, batch)
b = lambda i, prev_mention, indices, loss: [i + 1, tf.gather(mentions, i),
set_indices(i, prev_mention, indices),
loss + mention_group_loss(i, prev_mention, indices)]
loop = tf.while_loop(c, b, [i0, prev_mention0, indices0, loss0],
shape_invariants=[i0.get_shape(), prev_mention0.get_shape(),
tf.TensorShape([None]), loss0.get_shape()])
# c = lambda i, prev_mention, loss: tf.less(i, batch)
# b = lambda i, prev_mention, loss: [i + 1, tf.gather(mentions_arr, i), loss + mention_group_loss(i, prev_mention)]
# loop = tf.while_loop(c, b, [i0, prev_mention0, loss0])
# for i, mention in enumerate(mentions_arr):
#
# label = labels_arr[i]
# # label = tf.Print(label, [label], 'label: ')
# score = scores_arr[i]
# # score = tf.Print(score, [score], 'score: ')
#
# condition = tf.equal(mention, current_mention)
# # condition = tf.Print(condition, [condition], 'condition: ')
#
# loss.append(tf.cond(condition, lambda: same(label, score, i), lambda: different(label, score, i)))
# current_mention = mention
# mentions_arr = tf.Print(mentions_arr, [mentions_arr], "mentions_arr: ", first_n=1, summarize=100)
# labels_arr = tf.Print(labels_arr, [labels_arr], "labels_arr: ", first_n=1, summarize=100)
# scores_arr = tf.Print(scores_arr, [scores_arr], "scores_arr: ", first_n=1, summarize=100)
#
# # loss = tf.Print(loss, [loss], 'loss_arr: ', first_n=1, summarize=100)
# print_loop = tf.Print(loop, [loop[2]], 'loop vars: ')
# with tf.control_dependencies([print_loop]):
# return K.sum(loss)
return loop[3]
def sample_sequence(*,
hparams,
length,
start_token=None,
batch_size=None,
context=None,
temperature=1,
top_k=0):
if start_token is None:
assert context is not None, 'Specify exactly one of start_token and context!'
else:
assert context is None, 'Specify exactly one of start_token and context!'
context = tf.fill([batch_size, 1], start_token)
def step(hparams, tokens, past=None):
lm_output = model.model(hparams=hparams,
X=tokens,
past=past,
reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(
model.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
with tf.name_scope('sample_sequence'):
# Don't feed the last context token -- leave that to the loop below
# TODO: Would be slightly faster if we called step on the entire context,
# rather than leaving the last token transformer calculation to the while loop.
context_output = step(hparams, context[:, :-1])
def body(past, prev, output):
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
logits = next_outputs['logits'][:,
-1, :] / tf.to_float(temperature)
logits = top_k_logits(logits, k=top_k)
samples = tf.multinomial(logits,
num_samples=1,
output_dtype=tf.int32)
return [
tf.concat([past, next_outputs['presents']], axis=-2),
tf.squeeze(samples, axis=[1]),
tf.concat([output, samples], axis=1),
]
def cond(*args):
return True
_, _, tokens = tf.while_loop(
cond=cond,
body=body,
maximum_iterations=length,
loop_vars=[
context_output['presents'],
context[:, -1],
context,
],
shape_invariants=[
tf.TensorShape(
model.past_shape(hparams=hparams, batch_size=batch_size)),
tf.TensorShape([batch_size]),
tf.TensorShape([batch_size, None]),
],
back_prop=False,
)
return tokens
def _dynamic_unroll_multi_step(cell, inputs, reset_mask, initial_state,
zero_state, dtype, mask_fn, parallel_iterations,
swap_memory, iterations, batch_size,
const_batch_size):
"""Helper for dynamic_unroll which uses a tf.while_loop."""
# Convert all inputs to TensorArrays
def ta_and_unstack(x):
return (tf.TensorArray(dtype=x.dtype,
size=iterations,
element_shape=x.shape[1:]).unstack(x))
inputs_tas = nest.map_structure(ta_and_unstack, inputs)
reset_mask_ta = ta_and_unstack(reset_mask)
# Create a TensorArray for each output
def create_output_ta(s):
return tf.TensorArray(dtype=_infer_state_dtype(dtype, initial_state),
size=iterations,
element_shape=(tf.TensorShape(
[const_batch_size]).concatenate(
_maybe_tensor_shape_from_tensor(s))))
output_tas = nest.map_structure(create_output_ta, cell.output_size)
if mask_fn:
masks_ta = tf.TensorArray(dtype=tf.float32,
size=iterations,
element_shape=tf.TensorShape(
[const_batch_size]))
else:
masks_ta = ()
def pred(time, *unused_args):
return time < iterations
def body(time, time_since_reset, state, output_tas, masks_ta):
"""Internal while_loop body.
Args:
time: time
time_since_reset: time since last prev_time_steps.is_first() == true.
(only accurate / valid when mask_fn is not None).
state: rnn state @ time
output_tas: output tensorarrays
masks_ta: optional mask tensorarray
Returns:
- time + 1
- time_since_reset (next value)
- state: rnn state @ time + 1
- output_tas: output tensorarrays with values written @ time
- masks_ta: optional mask tensorarray with mask written @ time
"""
input_ = nest.map_structure(lambda ta: ta.read(time), inputs_tas)
is_reset = reset_mask_ta.read(time)
state = nest.map_structure(
lambda s_zero, s: _maybe_reset_state(is_reset, s_zero, s),
zero_state, state)
outputs, next_state = cell(input_, state)
output_tas = nest.map_structure(lambda ta, x: ta.write(time, x),
output_tas, outputs)
if mask_fn:
time_since_reset = tf.where(is_reset,
tf.zeros_like(time_since_reset),
time_since_reset + 1,
name="time_since_reset")
masks_ta = masks_ta.write(time, mask_fn(time, time_since_reset))
return (time + 1, time_since_reset, next_state, output_tas, masks_ta)
# Create a new scope in which the caching device is either
# determined by the parent scope, or is set to place the cached
# Variable using the same placement as for the rest of the RNN.
with tf.variable_scope(tf.get_variable_scope()) as varscope:
if (not tf.contrib.eager.executing_eagerly()
and varscope.caching_device is None):
varscope.set_caching_device(lambda op: op.device)
_, _, final_state, output_tas, masks_ta = (tf.while_loop(
pred,
body,
(tf.constant(0, name="time"),
tf.zeros((batch_size, ), dtype=tf.int32, name="time_since_reset"),
initial_state, output_tas, masks_ta),
parallel_iterations=parallel_iterations,
swap_memory=swap_memory,
maximum_iterations=iterations))
outputs = nest.map_structure(lambda ta: ta.stack(), output_tas)
if mask_fn:
mask = masks_ta.stack()
else:
mask = None
if isinstance(iterations, int):
# TensorArray.stack() doesn't set a static value for dimension 0,
# even if the size is known. Set the shapes here.
iterations_shape = tf.TensorShape([iterations])
for tensor in nest.flatten(outputs) + ([mask] if mask_fn else []):
tensor.set_shape(iterations_shape.concatenate(tensor.shape[1:]))
# Convert everything back to batch major
outputs = nest.map_structure(tf.contrib.rnn.transpose_batch_time, outputs)
if mask is not None:
mask = tf.transpose(mask)
return (outputs, final_state, mask)
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
"""Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
"""
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) + (
(1 - object_mask) * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) *
ignore_mask)
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss
if print_loss:
loss = tf.Print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
K.sum(ignore_mask)
],
message="loss: ")
return loss
def inference_mem(images,
cams,
depth_num,
depth_start,
depth_interval,
is_master_gpu=True):
""" infer depth image from multi-view images and cameras """
# dynamic gpu params
depth_end = depth_start + (tf.cast(depth_num, tf.float32) -
1) * depth_interval
feature_c = 32
feature_h = FLAGS.max_h / 4
feature_w = FLAGS.max_w / 4
# reference image
ref_image = tf.squeeze(tf.slice(images, [0, 0, 0, 0, 0],
[-1, 1, -1, -1, 3]),
axis=1)
ref_cam = tf.squeeze(tf.slice(cams, [0, 0, 0, 0, 0], [-1, 1, 2, 4, 4]),
axis=1)
# image feature extraction
if is_master_gpu:
ref_tower = UNetDS2GN({'data': ref_image},
is_training=True,
reuse=False)
else:
ref_tower = UNetDS2GN({'data': ref_image},
is_training=True,
reuse=True)
ref_feature = ref_tower.get_output()
ref_feature2 = tf.square(ref_feature)
view_features = []
for view in range(1, FLAGS.view_num):
view_image = tf.squeeze(tf.slice(images, [0, view, 0, 0, 0],
[-1, 1, -1, -1, -1]),
axis=1)
view_tower = UNetDS2GN({'data': view_image},
is_training=True,
reuse=True)
view_features.append(view_tower.get_output())
view_features = tf.stack(view_features, axis=0)
# get all homographies
view_homographies = []
for view in range(1, FLAGS.view_num):
view_cam = tf.squeeze(tf.slice(cams, [0, view, 0, 0, 0],
[-1, 1, 2, 4, 4]),
axis=1)
homographies = get_homographies(ref_cam,
view_cam,
depth_num=depth_num,
depth_start=depth_start,
depth_interval=depth_interval)
view_homographies.append(homographies)
view_homographies = tf.stack(view_homographies, axis=0)
# build cost volume by differentialble homography
with tf.name_scope('cost_volume_homography'):
depth_costs = []
for d in range(depth_num):
# compute cost (standard deviation feature)
ave_feature = tf.Variable(
tf.zeros([FLAGS.batch_size, feature_h, feature_w, feature_c]),
name='ave',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
ave_feature2 = tf.Variable(
tf.zeros([FLAGS.batch_size, feature_h, feature_w, feature_c]),
name='ave2',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
ave_feature = tf.assign(ave_feature, ref_feature)
ave_feature2 = tf.assign(ave_feature2, ref_feature2)
def body(view, ave_feature, ave_feature2):
"""Loop body."""
homography = tf.slice(view_homographies[view],
begin=[0, d, 0, 0],
size=[-1, 1, 3, 3])
homography = tf.squeeze(homography, axis=1)
# warped_view_feature = homography_warping(view_features[view], homography)
warped_view_feature = tf_transform_homography(
view_features[view], homography)
ave_feature = tf.assign_add(ave_feature, warped_view_feature)
ave_feature2 = tf.assign_add(ave_feature2,
tf.square(warped_view_feature))
view = tf.add(view, 1)
return view, ave_feature, ave_feature2
view = tf.constant(0)
cond = lambda view, *_: tf.less(view, FLAGS.view_num - 1)
_, ave_feature, ave_feature2 = tf.while_loop(
cond,
body, [view, ave_feature, ave_feature2],
back_prop=False,
parallel_iterations=1)
ave_feature = tf.assign(
ave_feature,
tf.square(ave_feature) / (FLAGS.view_num * FLAGS.view_num))
ave_feature2 = tf.assign(
ave_feature2, ave_feature2 / FLAGS.view_num - ave_feature)
depth_costs.append(ave_feature2)
cost_volume = tf.stack(depth_costs, axis=1)
# filtered cost volume, size of (B, D, H, W, 1)
if is_master_gpu:
filtered_cost_volume_tower = RegNetUS0({'data': cost_volume},
is_training=True,
reuse=False)
else:
filtered_cost_volume_tower = RegNetUS0({'data': cost_volume},
is_training=True,
reuse=True)
filtered_cost_volume = tf.squeeze(filtered_cost_volume_tower.get_output(),
axis=-1)
# depth map by softArgmin
with tf.name_scope('soft_arg_min'):
# probability volume by soft max
probability_volume = tf.nn.softmax(tf.scalar_mul(
-1, filtered_cost_volume),
axis=1,
name='prob_volume')
# depth image by soft argmin
volume_shape = tf.shape(probability_volume)
soft_2d = []
for i in range(FLAGS.batch_size):
soft_1d = tf.linspace(depth_start[i], depth_end[i],
tf.cast(depth_num, tf.int32))
soft_2d.append(soft_1d)
soft_2d = tf.reshape(tf.stack(soft_2d, axis=0),
[volume_shape[0], volume_shape[1], 1, 1])
soft_4d = tf.tile(soft_2d, [1, 1, volume_shape[2], volume_shape[3]])
estimated_depth_map = tf.reduce_sum(soft_4d * probability_volume,
axis=1)
estimated_depth_map = tf.expand_dims(estimated_depth_map, axis=3)
# probability map
prob_map = get_propability_map(probability_volume, estimated_depth_map,
depth_start, depth_interval)
# return filtered_depth_map,
return estimated_depth_map, prob_map
def ssd_crop(image, boxes, classes):
"""IoU biassed random crop.
Reference: https://github.com/chauhan-utk/ssd.DomainAdaptation
"""
num_boxes = tf.shape(boxes)[0]
def no_crop_check():
return (tf.random_uniform(shape=(), minval=0, maxval=1, dtype=tf.float32)
< ssd_constants.P_NO_CROP_PER_PASS)
def no_crop_proposal():
return (
tf.ones((), tf.bool),
tf.convert_to_tensor([0, 0, 1, 1], dtype=tf.float32),
tf.ones((num_boxes,), tf.bool),
)
def crop_proposal():
rand_vec = lambda minval, maxval: tf.random_uniform(
shape=(ssd_constants.NUM_CROP_PASSES, 1), minval=minval, maxval=maxval,
dtype=tf.float32)
width, height = rand_vec(0.3, 1), rand_vec(0.3, 1)
left, top = rand_vec(0, 1-width), rand_vec(0, 1-height)
right = left + width
bottom = top + height
ltrb = tf.concat([left, top, right, bottom], axis=1)
min_iou = tf.random_shuffle(ssd_constants.CROP_MIN_IOU_CHOICES)[0]
ious = calc_iou_tensor(ltrb, boxes)
# discard any bboxes whose center not in the cropped image
xc, yc = [tf.tile(0.5 * (boxes[:, i + 0] + boxes[:, i + 2])[tf.newaxis, :],
(ssd_constants.NUM_CROP_PASSES, 1)) for i in range(2)]
masks = tf.reduce_all(tf.stack([
tf.greater(xc, tf.tile(left, (1, num_boxes))),
tf.less(xc, tf.tile(right, (1, num_boxes))),
tf.greater(yc, tf.tile(top, (1, num_boxes))),
tf.less(yc, tf.tile(bottom, (1, num_boxes))),
], axis=2), axis=2)
# Checks of whether a crop is valid.
valid_aspect = tf.logical_and(tf.less(height/width, 2),
tf.less(height/width, 2))
valid_ious = tf.reduce_all(tf.greater(ious, min_iou), axis=1, keepdims=True)
valid_masks = tf.reduce_any(masks, axis=1, keepdims=True)
valid_all = tf.cast(tf.reduce_all(tf.concat(
[valid_aspect, valid_ious, valid_masks], axis=1), axis=1), tf.int32)
# One indexed, as zero is needed for the case of no matches.
index = tf.range(1, 1 + ssd_constants.NUM_CROP_PASSES, dtype=tf.int32)
# Either one-hot, or zeros if there is no valid crop.
selection = tf.equal(tf.reduce_max(index * valid_all), index)
use_crop = tf.reduce_any(selection)
output_ltrb = tf.reduce_sum(tf.multiply(ltrb, tf.tile(tf.cast(
selection, tf.float32)[:, tf.newaxis], (1, 4))), axis=0)
output_masks = tf.reduce_any(tf.logical_and(masks, tf.tile(
selection[:, tf.newaxis], (1, num_boxes))), axis=0)
return use_crop, output_ltrb, output_masks
def proposal(*args):
return tf.cond(
pred=no_crop_check(),
true_fn=no_crop_proposal,
false_fn=crop_proposal,
)
_, crop_bounds, box_masks = tf.while_loop(
cond=lambda x, *_: tf.logical_not(x),
body=proposal,
loop_vars=[tf.zeros((), tf.bool), tf.zeros((4,), tf.float32), tf.zeros((num_boxes,), tf.bool)],
)
filtered_boxes = tf.boolean_mask(boxes, box_masks, axis=0)
num_filtered_boxes = tf.shape(filtered_boxes)[0]
# Clip boxes to the cropped region.
filtered_boxes = tf.stack([
tf.maximum(filtered_boxes[:, 0], crop_bounds[0]),
tf.maximum(filtered_boxes[:, 1], crop_bounds[1]),
tf.minimum(filtered_boxes[:, 2], crop_bounds[2]),
tf.minimum(filtered_boxes[:, 3], crop_bounds[3]),
], axis=1)
left = crop_bounds[0]
top = crop_bounds[1]
width = crop_bounds[2] - left
height = crop_bounds[3] - top
cropped_boxes = tf.stack([
(filtered_boxes[:, 0] - left) / width,
(filtered_boxes[:, 1] - top) / height,
(filtered_boxes[:, 2] - left) / width,
(filtered_boxes[:, 3] - top) / height,
], axis=1)
cropped_image = tf.image.crop_and_resize(
image=image[tf.newaxis, :, :, :],
boxes=crop_bounds[tf.newaxis, :],
box_ind=tf.zeros((1,), tf.int32),
crop_size=(ssd_constants.IMAGE_SIZE, ssd_constants.IMAGE_SIZE),
)[0, :, :, :]
cropped_classes = tf.boolean_mask(classes, box_masks, axis=0)
return cropped_image, cropped_boxes, cropped_classes
def beam_search(symbols_to_logits_fn,
initial_ids,
beam_size,
decode_length,
vocab_size,
alpha,
states=None,
eos_id=EOS_ID):
"""Beam search with length penalties.
Requires a function that can take the currently decoded sybmols and return
the logits for the next symbol. The implementation is inspired by
https://arxiv.org/abs/1609.08144.
When running, the beam search steps can be visualized by using tfdbg to watch
the operations generating the output ids for each beam step. These operations
have the pattern:
(alive|finished)_topk_(seq,scores)
Operations marked `alive` represent the new beam sequences that will be
processed in the next step. Operations marked `finished` represent the
completed beam sequences, which may be padded with 0s if no beams finished.
Operations marked `seq` store the full beam sequence for the time step.
Operations marked `scores` store the sequence's final log scores.
The beam search steps will be processed sequentially in order, so when
capturing observed from these operations, tensors, clients can make
assumptions about which step is being recorded.
Args:
symbols_to_logits_fn: Interface to the model, to provide logits.
Shoud take [batch_size, decoded_ids] and return [batch_size, vocab_size]
initial_ids: Ids to start off the decoding, this will be the first thing
handed to symbols_to_logits_fn (after expanding to beam size)
[batch_size]
beam_size: Size of the beam.
decode_length: Number of steps to decode for.
vocab_size: Size of the vocab, must equal the size of the logits returned by
symbols_to_logits_fn
alpha: alpha for length penalty.
eos_id: ID for end of sentence.
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(decoded beams [batch_size, beam_size, decode_length]
decoding probablities [batch_size, beam_size])
"""
batch_size = tf.shape(initial_ids)[0]
# Assume initial_ids are prob 1.0
initial_log_probs = tf.constant([[0.] + [-float("inf")] * (beam_size - 1)])
# Expand to beam_size (batch_size, beam_size)
alive_log_probs = tf.tile(initial_log_probs, [batch_size, 1])
# Expand each batch and state to beam_size
alive_seq = expand_to_beam_size(initial_ids, beam_size)
alive_seq = tf.expand_dims(alive_seq, axis=2) # (batch_size, beam_size, 1)
if states:
states = nest.map_structure(
lambda state: expand_to_beam_size(state, beam_size), states)
else:
states = {}
# Finished will keep track of all the sequences that have finished so far
# Finished log probs will be negative infinity in the beginning
# finished_flags will keep track of booleans
finished_seq = tf.zeros(tf.shape(alive_seq), tf.int32)
# Setting the scores of the initial to negative infinity.
finished_scores = tf.ones([batch_size, beam_size]) * -INF
finished_flags = tf.zeros([batch_size, beam_size], tf.bool)
def grow_finished(finished_seq, finished_scores, finished_flags, curr_seq,
curr_scores, curr_finished):
"""Given sequences and scores, will gather the top k=beam size sequences.
Args:
finished_seq: Current finished sequences.
[batch_size, beam_size, current_decoded_length]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_flags: finished bools for each of these sequences.
[batch_size, beam_size]
curr_seq: current topk sequence that has been grown by one position.
[batch_size, beam_size, current_decoded_length]
curr_scores: scores for each of these sequences. [batch_size, beam_size]
curr_finished: Finished flags for each of these sequences.
[batch_size, beam_size]
Returns:
Tuple of
(Topk sequences based on scores,
log probs of these sequences,
Finished flags of these sequences)
"""
# First append a column of 0'ids to finished to make the same length with
# finished scores
finished_seq = tf.concat(
[finished_seq,
tf.zeros([batch_size, beam_size, 1], tf.int32)],
axis=2)
# Set the scores of the unfinished seq in curr_seq to large negative
# values
curr_scores += (1. - tf.to_float(curr_finished)) * -INF
# concatenating the sequences and scores along beam axis
curr_finished_seq = tf.concat([finished_seq, curr_seq], axis=1)
curr_finished_scores = tf.concat([finished_scores, curr_scores],
axis=1)
curr_finished_flags = tf.concat([finished_flags, curr_finished],
axis=1)
return compute_topk_scores_and_seq(curr_finished_seq,
curr_finished_scores,
curr_finished_scores,
curr_finished_flags, beam_size,
batch_size, "grow_finished")
def grow_alive(curr_seq, curr_scores, curr_log_probs, curr_finished,
states):
"""Given sequences and scores, will gather the top k=beam size sequences.
Args:
curr_seq: current topk sequence that has been grown by one position.
[batch_size, beam_size, i+1]
curr_scores: scores for each of these sequences. [batch_size, beam_size]
curr_log_probs: log probs for each of these sequences.
[batch_size, beam_size]
curr_finished: Finished flags for each of these sequences.
[batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Topk sequences based on scores,
log probs of these sequences,
Finished flags of these sequences)
"""
# Set the scores of the finished seq in curr_seq to large negative
# values
curr_scores += tf.to_float(curr_finished) * -INF
return compute_topk_scores_and_seq(curr_seq, curr_scores,
curr_log_probs, curr_finished,
beam_size, batch_size, "grow_alive",
states)
def grow_topk(i, alive_seq, alive_log_probs, states):
r"""Inner beam seach loop.
This function takes the current alive sequences, and grows them to topk
sequences where k = 2*beam. We use 2*beam because, we could have beam_size
number of sequences that might hit <EOS> and there will be no alive
sequences to continue. With 2*beam_size, this will not happen. This relies
on the assumption the vocab size is > beam size. If this is true, we'll
have at least beam_size non <EOS> extensions if we extract the next top
2*beam words.
Length penalty is given by = (5+len(decode)/6) ^ -\alpha. Pls refer to
https://arxiv.org/abs/1609.08144.
Args:
i: loop index
alive_seq: Topk sequences decoded so far [batch_size, beam_size, i+1]
alive_log_probs: probabilities of these sequences. [batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Topk sequences extended by the next word,
The log probs of these sequences,
The scores with length penalty of these sequences,
Flags indicating which of these sequences have finished decoding,
dict of transformed decoding states)
"""
# Get the logits for all the possible next symbols
flat_ids = tf.reshape(alive_seq, [batch_size * beam_size, -1])
# (batch_size * beam_size, decoded_length)
if states:
flat_states = nest.map_structure(
lambda state: tf.reshape(state, [batch_size * beam_size, -1]),
states)
flat_logits, flat_states = symbols_to_logits_fn(
flat_ids, flat_states)
states = nest.map_structure(
lambda state: tf.reshape(state, [batch_size, beam_size, -1]),
flat_states)
else:
flat_logits = symbols_to_logits_fn(flat_ids)
logits = tf.reshape(flat_logits, (batch_size, beam_size, -1))
# Convert logits to normalized log probs
candidate_log_probs = log_prob_from_logits(logits)
# Multiply the probabilites by the current probabilites of the beam.
# (batch_size, beam_size, vocab_size) + (batch_size, beam_size, 1)
log_probs = candidate_log_probs + tf.expand_dims(alive_log_probs,
axis=2)
length_penalty = tf.pow(((5. + tf.to_float(i + 1)) / 6.), alpha)
curr_scores = log_probs / length_penalty
# Flatten out (beam_size, vocab_size) probs in to a list of possibilites
flat_curr_scores = tf.reshape(curr_scores,
[-1, beam_size * vocab_size])
topk_scores, topk_ids = tf.nn.top_k(flat_curr_scores, k=beam_size * 2)
# Recovering the log probs because we will need to send them back
topk_log_probs = topk_scores * length_penalty
# Work out what beam the top probs are in.
topk_beam_index = topk_ids // vocab_size
topk_ids %= vocab_size # Unflatten the ids
# The next three steps are to create coordinates for tf.gather_nd to pull
# out the correct seqences from id's that we need to grow.
# We will also use the coordinates to gather the booleans of the beam items
# that survived.
batch_pos = compute_batch_indices(batch_size, beam_size * 2)
# top beams will give us the actual coordinates to do the gather.
# stacking will create a tensor of dimension batch * beam * 2, where the
# last dimension contains the i,j gathering coordinates.
topk_coordinates = tf.stack([batch_pos, topk_beam_index], axis=2)
# Gather up the most probable 2*beams both for the ids and finished_in_alive
# bools
topk_seq = tf.gather_nd(alive_seq, topk_coordinates)
if states:
states = nest.map_structure(
lambda state: tf.gather_nd(state, topk_coordinates), states)
# Append the most probable alive
topk_seq = tf.concat(
[topk_seq, tf.expand_dims(topk_ids, axis=2)], axis=2)
topk_finished = tf.equal(topk_ids, eos_id)
return topk_seq, topk_log_probs, topk_scores, topk_finished, states
def inner_loop(i, alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states):
"""Inner beam seach loop.
There are three groups of tensors, alive, finished, and topk.
The alive group contains information about the current alive sequences
The topk group contains information about alive + topk current decoded words
the finished group contains information about finished sentences, that is,
the ones that have decoded to <EOS>. These are what we return.
The general beam search algorithm is as follows:
While we haven't terminated (pls look at termination condition)
1. Grow the current alive to get beam*2 topk sequences
2. Among the topk, keep the top beam_size ones that haven't reached EOS
into alive
3. Among the topk, keep the top beam_size ones have reached EOS into
finished
Repeat
To make things simple with using fixed size tensors, we will end
up inserting unfinished sequences into finished in the beginning. To stop
that we add -ve INF to the score of the unfinished sequence so that when a
true finished sequence does appear, it will have a higher score than all the
unfinished ones.
Args:
i: loop index
alive_seq: Topk sequences decoded so far [batch_size, beam_size, i+1]
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_seq: Current finished sequences.
[batch_size, beam_size, i+1]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_flags: finished bools for each of these sequences.
[batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Incremented loop index
New alive sequences,
Log probs of the alive sequences,
New finished sequences,
Scores of the new finished sequences,
Flags inidicating which sequence in finished as reached EOS,
dict of final decoding states)
"""
# Each inner loop, we carry out three steps:
# 1. Get the current topk items.
# 2. Extract the ones that have finished and haven't finished
# 3. Recompute the contents of finished based on scores.
topk_seq, topk_log_probs, topk_scores, topk_finished, states = grow_topk(
i, alive_seq, alive_log_probs, states)
alive_seq, alive_log_probs, _, states = grow_alive(
topk_seq, topk_scores, topk_log_probs, topk_finished, states)
finished_seq, finished_scores, finished_flags, _ = grow_finished(
finished_seq, finished_scores, finished_flags, topk_seq,
topk_scores, topk_finished)
return (i + 1, alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states)
def _is_finished(i, unused_alive_seq, alive_log_probs, unused_finished_seq,
finished_scores, finished_in_finished, unused_states):
"""Checking termination condition.
We terminate when we decoded up to decode_length or the lowest scoring item
in finished has a greater score that the higest prob item in alive divided
by the max length penalty
Args:
i: loop index
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_in_finished: finished bools for each of these sequences.
[batch_size, beam_size]
Returns:
Bool.
"""
max_length_penalty = tf.pow(((5. + tf.to_float(decode_length)) / 6.),
alpha)
# The best possible score of the most likley alive sequence
lower_bound_alive_scores = alive_log_probs[:, 0] / max_length_penalty
# Now to compute the lowest score of a finished sequence in finished
# If the sequence isn't finished, we multiply it's score by 0. since
# scores are all -ve, taking the min will give us the score of the lowest
# finished item.
lowest_score_of_fininshed_in_finished = tf.reduce_min(
finished_scores * tf.to_float(finished_in_finished), axis=1)
# If none of the sequences have finished, then the min will be 0 and
# we have to replace it by -ve INF if it is. The score of any seq in alive
# will be much higher than -ve INF and the termination condition will not
# be met.
lowest_score_of_fininshed_in_finished += (
(1. - tf.to_float(tf.reduce_any(finished_in_finished, 1))) * -INF)
bound_is_met = tf.reduce_all(
tf.greater(lowest_score_of_fininshed_in_finished,
lower_bound_alive_scores))
return tf.logical_and(tf.less(i, decode_length),
tf.logical_not(bound_is_met))
(_, alive_seq, alive_log_probs, finished_seq, finished_scores,
finished_flags, _) = tf.while_loop(
_is_finished,
inner_loop, [
tf.constant(0), alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states
],
shape_invariants=[
tf.TensorShape([]),
tf.TensorShape([None, None, None]),
alive_log_probs.get_shape(),
tf.TensorShape([None, None, None]),
finished_scores.get_shape(),
finished_flags.get_shape(),
nest.map_structure(
lambda tensor: tf.TensorShape([None] * tensor.shape.ndims),
states),
],
parallel_iterations=1,
back_prop=False)
alive_seq.set_shape((None, beam_size, None))
finished_seq.set_shape((None, beam_size, None))
# Accounting for corner case: It's possible that no sequence in alive for a
# particular batch item ever reached EOS. In that case, we should just copy
# the contents of alive for that batch item. tf.reduce_any(finished_flags, 1)
# if 0, means that no sequence for that batch index had reached EOS. We need
# to do the same for the scores as well.
finished_seq = tf.where(tf.reduce_any(finished_flags, 1), finished_seq,
alive_seq)
finished_scores = tf.where(tf.reduce_any(finished_flags, 1),
finished_scores, alive_log_probs)
return finished_seq, finished_scores
def structural_loss(fake1, fake2):
eps = 1e-4
win_rad = 1
win_size = (win_rad * 2 + 1)**2
b, d, h, w = fake1.get_shape().as_list()
c_h, c_w = h - 2 * win_rad, w - 2 * win_rad
win_diam = win_rad * 2 + 1
indsM = tf.reshape(tf.range(h * w), [1, h, w, 1])
win_ids = extract_patches(indsM, win_rad * 2 + 1, h, w)
A = tf.reshape(win_ids, [-1, 9, 1])
A = tf.tile(A, [1, 1, 9])
col = tf.transpose(A, [0, 2, 1])
col = tf.cast(tf.reshape(col, [-1]), tf.int64)
row = tf.cast(tf.reshape(A, [-1]), tf.int64)
def laplacian_matrix(img):
temp = tf.transpose(img, [0, 2, 3, 1])
winI = extract_patches(temp, win_rad * 2 + 1, h, w)
winI = tf.reshape(winI, [-1, (h - 2) * (w - 2), 9, 3])
win_mu = tf.reduce_mean(winI, axis=2, keep_dims=True)
term1 = tf.matmul(tf.transpose(winI, [0, 1, 3, 2]), winI)
term2 = tf.matmul(tf.transpose(win_mu, [0, 1, 3, 2]), win_mu)
win_var = term1 / win_size - term2
inv = tf.linalg.inv(win_var + (eps / win_size) * tf.eye(3))
X = tf.matmul(winI - win_mu, inv)
vals = tf.eye(win_size) - (1.0 / win_size) * (
1 + tf.matmul(X, tf.transpose(winI - win_mu, [0, 1, 3, 2])))
vals = tf.layers.flatten(vals)
SM = tf.SparseTensor(indices=tf.stack([row, col], 1),
values=vals[0],
dense_shape=[h * w, h * w])
return SM
def condition(i, loss):
return tf.less(i, tf.shape(fake1)[0])
def action(i, loss):
slice1 = fake1[i:i + 1]
slice2 = fake2[i:i + 1]
L1 = laplacian_matrix(slice1)
L2 = laplacian_matrix(slice2)
size = fake1.get_shape().as_list()[2]
temp = tf.reshape(slice1, [3, -1])
covariance = tf.matmul(
temp, tf.sparse_tensor_dense_matmul(L2,
tf.transpose(temp))) / size**2
str_loss_1 = tf.trace(covariance)
temp = tf.reshape(slice2, [3, -1])
covariance = tf.matmul(
temp, tf.sparse_tensor_dense_matmul(L1,
tf.transpose(temp))) / size**2
str_loss_2 = tf.trace(covariance)
str_loss_1 = tf.reshape(str_loss_1, [1, 1])
str_loss_2 = tf.reshape(str_loss_2, [1, 1])
loss = tf.concat([loss, str_loss_1, str_loss_2], axis=0)
return tf.add(i, 1), loss
i = tf.constant(0)
loss = tf.Variable(np.zeros((0, 1), dtype=np.float32))
final_index, loss = tf.while_loop(
condition,
action, [i, loss],
shape_invariants=[i.get_shape(),
tf.TensorShape([None, 1])])
return loss
def sample_annealed_importance_chain(num_steps,
proposal_log_prob_fn,
target_log_prob_fn,
current_state,
make_kernel_fn,
parallel_iterations=10,
name=None):
"""Runs annealed importance sampling (AIS) to estimate normalizing constants.
This function uses Hamiltonian Monte Carlo to sample from a series of
distributions that slowly interpolates between an initial "proposal"
distribution:
`exp(proposal_log_prob_fn(x) - proposal_log_normalizer)`
and the target distribution:
`exp(target_log_prob_fn(x) - target_log_normalizer)`,
accumulating importance weights along the way. The product of these
importance weights gives an unbiased estimate of the ratio of the
normalizing constants of the initial distribution and the target
distribution:
`E[exp(ais_weights)] = exp(target_log_normalizer - proposal_log_normalizer)`.
Note: `proposal_log_prob_fn` and `target_log_prob_fn` are called exactly three
times (although this may be reduced to two times, in the future).
Args:
num_steps: Integer number of Markov chain updates to run. More
iterations means more expense, but smoother annealing between q
and p, which in turn means exponentially lower variance for the
normalizing constant estimator.
proposal_log_prob_fn: Python callable that returns the log density of the
initial distribution.
target_log_prob_fn: Python callable which takes an argument like
`current_state` (or `*current_state` if it's a list) and returns its
(possibly unnormalized) log-density under the target distribution.
current_state: `Tensor` or Python `list` of `Tensor`s representing the
current state(s) of the Markov chain(s). The first `r` dimensions index
independent chains, `r = tf.rank(target_log_prob_fn(*current_state))`.
make_kernel_fn: Python `callable` which returns a `TransitionKernel`-like
object. Must take one argument representing the `TransitionKernel`'s
`target_log_prob_fn`. The `target_log_prob_fn` argument represents the
`TransitionKernel`'s target log distribution. Note:
`sample_annealed_importance_chain` creates a new `target_log_prob_fn`
which
is an interpolation between the supplied `target_log_prob_fn` and
`proposal_log_prob_fn`; it is this interpolated function which is used as an
argument to `make_kernel_fn`.
parallel_iterations: The number of iterations allowed to run in parallel.
It must be a positive integer. See `tf.while_loop` for more details.
name: Python `str` name prefixed to Ops created by this function.
Default value: `None` (i.e., "sample_annealed_importance_chain").
Returns:
next_state: `Tensor` or Python list of `Tensor`s representing the
state(s) of the Markov chain(s) at the final iteration. Has same shape as
input `current_state`.
ais_weights: Tensor with the estimated weight(s). Has shape matching
`target_log_prob_fn(current_state)`.
kernel_results: `collections.namedtuple` of internal calculations used to
advance the chain.
#### Examples
##### Estimate the normalizing constant of a log-gamma distribution.
```python
tfd = tf.contrib.distributions
# Run 100 AIS chains in parallel
num_chains = 100
dims = 20
dtype = np.float32
proposal = tfd.MultivatiateNormalDiag(
loc=tf.zeros([dims], dtype=dtype))
target = tfd.TransformedDistribution(
distribution=tfd.Gamma(concentration=dtype(2),
rate=dtype(3)),
bijector=tfd.bijectors.Invert(tfd.bijectors.Exp()),
event_shape=[dims])
chains_state, ais_weights, kernels_results = (
tfp.mcmc.sample_annealed_importance_chain(
num_steps=1000,
proposal_log_prob_fn=proposal.log_prob,
target_log_prob_fn=target.log_prob,
current_state=proposal.sample(num_chains),
make_kernel_fn=lambda tlp_fn: tfp.mcmc.HamiltonianMonteCarlo(
target_log_prob_fn=tlp_fn,
step_size=0.2,
num_leapfrog_steps=2)))
log_estimated_normalizer = (tf.reduce_logsumexp(ais_weights)
- np.log(num_chains))
log_true_normalizer = tf.lgamma(2.) - 2. * tf.log(3.)
```
##### Estimate marginal likelihood of a Bayesian regression model.
```python
tfd = tf.contrib.distributions
def make_prior(dims, dtype):
return tfd.MultivariateNormalDiag(
loc=tf.zeros(dims, dtype))
def make_likelihood(weights, x):
return tfd.MultivariateNormalDiag(
loc=tf.tensordot(weights, x, axes=[[0], [-1]]))
# Run 100 AIS chains in parallel
num_chains = 100
dims = 10
dtype = np.float32
# Make training data.
x = np.random.randn(num_chains, dims).astype(dtype)
true_weights = np.random.randn(dims).astype(dtype)
y = np.dot(x, true_weights) + np.random.randn(num_chains)
# Setup model.
prior = make_prior(dims, dtype)
def target_log_prob_fn(weights):
return prior.log_prob(weights) + make_likelihood(weights, x).log_prob(y)
proposal = tfd.MultivariateNormalDiag(
loc=tf.zeros(dims, dtype))
weight_samples, ais_weights, kernel_results = (
tfp.mcmc.sample_annealed_importance_chain(
num_steps=1000,
proposal_log_prob_fn=proposal.log_prob,
target_log_prob_fn=target_log_prob_fn
current_state=tf.zeros([num_chains, dims], dtype),
make_kernel_fn=lambda tlp_fn: tfp.mcmc.HamiltonianMonteCarlo(
target_log_prob_fn=tlp_fn,
step_size=0.1,
num_leapfrog_steps=2)))
log_normalizer_estimate = (tf.reduce_logsumexp(ais_weights)
- np.log(num_chains))
```
"""
with tf.name_scope(name, "sample_annealed_importance_chain",
[num_steps, current_state]):
num_steps = tf.convert_to_tensor(num_steps,
dtype=tf.int32,
name="num_steps")
if mcmc_util.is_list_like(current_state):
current_state = [
tf.convert_to_tensor(s, name="current_state")
for s in current_state
]
else:
current_state = tf.convert_to_tensor(current_state,
name="current_state")
def _make_convex_combined_log_prob_fn(iter_):
def _fn(*args):
p = tf.identity(proposal_log_prob_fn(*args),
name="proposal_log_prob")
t = tf.identity(target_log_prob_fn(*args),
name="target_log_prob")
dtype = p.dtype.base_dtype
beta = tf.cast(iter_ + 1, dtype) / tf.cast(num_steps, dtype)
return tf.identity(beta * t + (1. - beta) * p,
name="convex_combined_log_prob")
return _fn
def _loop_body(iter_, ais_weights, current_state, kernel_results):
"""Closure which implements `tf.while_loop` body."""
x = (current_state
if mcmc_util.is_list_like(current_state) else [current_state])
proposal_log_prob = proposal_log_prob_fn(*x)
target_log_prob = target_log_prob_fn(*x)
ais_weights += ((target_log_prob - proposal_log_prob) /
tf.cast(num_steps, ais_weights.dtype))
kernel = make_kernel_fn(_make_convex_combined_log_prob_fn(iter_))
next_state, inner_results = kernel.one_step(
current_state, kernel_results.inner_results)
kernel_results = AISResults(
proposal_log_prob=proposal_log_prob,
target_log_prob=target_log_prob,
inner_results=inner_results,
)
return [iter_ + 1, ais_weights, next_state, kernel_results]
def _bootstrap_results(init_state):
"""Creates first version of `previous_kernel_results`."""
kernel = make_kernel_fn(_make_convex_combined_log_prob_fn(iter_=0))
inner_results = kernel.bootstrap_results(init_state)
convex_combined_log_prob = inner_results.accepted_results.target_log_prob
dtype = convex_combined_log_prob.dtype.as_numpy_dtype
shape = tf.shape(convex_combined_log_prob)
proposal_log_prob = tf.fill(shape,
dtype(np.nan),
name="bootstrap_proposal_log_prob")
target_log_prob = tf.fill(shape,
dtype(np.nan),
name="target_target_log_prob")
return AISResults(
proposal_log_prob=proposal_log_prob,
target_log_prob=target_log_prob,
inner_results=inner_results,
)
previous_kernel_results = _bootstrap_results(current_state)
inner_results = previous_kernel_results.inner_results
ais_weights = tf.zeros(shape=tf.broadcast_dynamic_shape(
tf.shape(inner_results.proposed_results.target_log_prob),
tf.shape(inner_results.accepted_results.target_log_prob)),
dtype=inner_results.proposed_results.
target_log_prob.dtype.base_dtype)
[_, ais_weights, current_state, kernel_results] = tf.while_loop(
cond=lambda iter_, *args: iter_ < num_steps,
body=_loop_body,
loop_vars=[
np.int32(0), # iter_
ais_weights,
current_state,
previous_kernel_results,
],
parallel_iterations=parallel_iterations)
return [current_state, ais_weights, kernel_results]
def dynamic_decode(decoder: Union[Decoder, BaseDecoder],
output_time_major: bool = False,
impute_finished: bool = False,
maximum_iterations: Optional[TensorLike] = None,
parallel_iterations: int = 32,
swap_memory: bool = False,
training: Optional[bool] = None,
scope: Optional[str] = None,
**kwargs) -> Tuple[Any, Any, Any]:
"""Perform dynamic decoding with `decoder`.
Calls initialize() once and step() repeatedly on the Decoder object.
Args:
decoder: A `Decoder` instance.
output_time_major: Python boolean. Default: `False` (batch major). If
`True`, outputs are returned as time major tensors (this mode is
faster). Otherwise, outputs are returned as batch major tensors (this
adds extra time to the computation).
impute_finished: Python boolean. If `True`, then states for batch
entries which are marked as finished get copied through and the
corresponding outputs get zeroed out. This causes some slowdown at
each time step, but ensures that the final state and outputs have
the correct values and that backprop ignores time steps that were
marked as finished.
maximum_iterations: `int32` scalar, maximum allowed number of decoding
steps. Default is `None` (decode until the decoder is fully done).
parallel_iterations: Argument passed to `tf.while_loop`.
swap_memory: Argument passed to `tf.while_loop`.
training: Python boolean. Indicates whether the layer should behave
in training mode or in inference mode. Only relevant
when `dropout` or `recurrent_dropout` is used.
scope: Optional variable scope to use.
**kwargs: dict, other keyword arguments for dynamic_decode. It might
contain arguments for `BaseDecoder` to initialize, which takes all
tensor inputs during call().
Returns:
`(final_outputs, final_state, final_sequence_lengths)`.
Raises:
ValueError: if `maximum_iterations` is provided but is not a scalar.
"""
with tf.compat.v1.variable_scope(scope, "decoder") as varscope:
# Determine context types.
ctxt = tf.compat.v1.get_default_graph()._get_control_flow_context()
is_xla = control_flow_util.GetContainingXLAContext(ctxt) is not None
in_while_loop = control_flow_util.GetContainingWhileContext(
ctxt) is not None
# Properly cache variable values inside the while_loop.
# Don't set a caching device when running in a loop, since it is
# possible that train steps could be wrapped in a tf.while_loop. In that
# scenario caching prevents forward computations in loop iterations from
# re-reading the updated weights.
if not tf.executing_eagerly() and not in_while_loop:
if varscope.caching_device is None:
varscope.set_caching_device(lambda op: op.device)
if maximum_iterations is not None:
maximum_iterations = tf.convert_to_tensor(
maximum_iterations, dtype=tf.int32, name="maximum_iterations")
if maximum_iterations.shape.ndims != 0:
raise ValueError("maximum_iterations must be a scalar")
if isinstance(decoder, Decoder):
initial_finished, initial_inputs, initial_state = decoder.initialize(
)
else:
# For BaseDecoder that takes tensor inputs during call.
decoder_init_input = kwargs.pop("decoder_init_input", None)
decoder_init_kwargs = kwargs.pop("decoder_init_kwargs", {})
initial_finished, initial_inputs, initial_state = decoder.initialize(
decoder_init_input, **decoder_init_kwargs)
zero_outputs = tf.nest.map_structure(
lambda shape, dtype: tf.zeros(
_prepend_batch(decoder.batch_size, shape), dtype=dtype),
decoder.output_size,
decoder.output_dtype,
)
if is_xla and maximum_iterations is None:
raise ValueError(
"maximum_iterations is required for XLA compilation.")
if maximum_iterations is not None:
initial_finished = tf.logical_or(initial_finished,
0 >= maximum_iterations)
initial_sequence_lengths = tf.zeros_like(initial_finished,
dtype=tf.int32)
initial_time = tf.constant(0, dtype=tf.int32)
def _shape(batch_size, from_shape):
if not isinstance(from_shape,
tf.TensorShape) or from_shape.ndims == 0:
return None
else:
batch_size = tf.get_static_value(
tf.convert_to_tensor(batch_size, name="batch_size"))
return tf.TensorShape([batch_size]).concatenate(from_shape)
dynamic_size = maximum_iterations is None or not is_xla
def _create_ta(s, d):
return tf.TensorArray(
dtype=d,
size=0 if dynamic_size else maximum_iterations,
dynamic_size=dynamic_size,
element_shape=_shape(decoder.batch_size, s),
)
initial_outputs_ta = tf.nest.map_structure(_create_ta,
decoder.output_size,
decoder.output_dtype)
def condition(
unused_time,
unused_outputs_ta,
unused_state,
unused_inputs,
finished,
unused_sequence_lengths,
):
return tf.logical_not(tf.reduce_all(finished))
def body(time, outputs_ta, state, inputs, finished, sequence_lengths):
"""Internal while_loop body.
Args:
time: scalar int32 tensor.
outputs_ta: structure of TensorArray.
state: (structure of) state tensors and TensorArrays.
inputs: (structure of) input tensors.
finished: bool tensor (keeping track of what's finished).
sequence_lengths: int32 tensor (keeping track of time of finish).
Returns:
`(time + 1, outputs_ta, next_state, next_inputs, next_finished,
next_sequence_lengths)`.
```
"""
(next_outputs, decoder_state, next_inputs,
decoder_finished) = decoder.step(time, inputs, state, training)
decoder_state_sequence_lengths = False
if decoder.tracks_own_finished:
next_finished = decoder_finished
lengths = getattr(decoder_state, "lengths", None)
if lengths is not None:
# sequence lengths are provided by decoder_state.lengths;
# overwrite our sequence lengths.
decoder_state_sequence_lengths = True
sequence_lengths = tf.cast(lengths, tf.int32)
else:
next_finished = tf.logical_or(decoder_finished, finished)
if decoder_state_sequence_lengths:
# Just pass something through the loop; at the next iteration
# we'll pull the sequence lengths from the decoder_state again.
next_sequence_lengths = sequence_lengths
else:
next_sequence_lengths = tf.where(
tf.logical_not(finished),
tf.fill(tf.shape(sequence_lengths), time + 1),
sequence_lengths,
)
tf.nest.assert_same_structure(state, decoder_state)
tf.nest.assert_same_structure(outputs_ta, next_outputs)
tf.nest.assert_same_structure(inputs, next_inputs)
# Zero out output values past finish
if impute_finished:
def zero_out_finished(out, zero):
if finished.shape.rank < zero.shape.rank:
broadcast_finished = tf.broadcast_to(
tf.expand_dims(finished, axis=-1), zero.shape)
return tf.where(broadcast_finished, zero, out)
else:
return tf.where(finished, zero, out)
emit = tf.nest.map_structure(zero_out_finished, next_outputs,
zero_outputs)
else:
emit = next_outputs
# Copy through states past finish
def _maybe_copy_state(new, cur):
# TensorArrays and scalar states get passed through.
if isinstance(cur, tf.TensorArray):
pass_through = True
else:
new.set_shape(cur.shape)
pass_through = new.shape.ndims == 0
if not pass_through:
broadcast_finished = tf.broadcast_to(
tf.expand_dims(finished, axis=-1), new.shape)
return tf.where(broadcast_finished, cur, new)
else:
return new
if impute_finished:
next_state = tf.nest.map_structure(_maybe_copy_state,
decoder_state, state)
else:
next_state = decoder_state
outputs_ta = tf.nest.map_structure(
lambda ta, out: ta.write(time, out), outputs_ta, emit)
return (
time + 1,
outputs_ta,
next_state,
next_inputs,
next_finished,
next_sequence_lengths,
)
res = tf.while_loop(
condition,
body,
loop_vars=(
initial_time,
initial_outputs_ta,
initial_state,
initial_inputs,
initial_finished,
initial_sequence_lengths,
),
parallel_iterations=parallel_iterations,
maximum_iterations=maximum_iterations,
swap_memory=swap_memory,
)
final_outputs_ta = res[1]
final_state = res[2]
final_sequence_lengths = res[5]
final_outputs = tf.nest.map_structure(lambda ta: ta.stack(),
final_outputs_ta)
try:
final_outputs, final_state = decoder.finalize(
final_outputs, final_state, final_sequence_lengths)
except NotImplementedError:
pass
if not output_time_major:
final_outputs = tf.nest.map_structure(_transpose_batch_time,
final_outputs)
return final_outputs, final_state, final_sequence_lengths
def __init__(self, batch_size, vocab_size, embedding_dim, hidden_size, max_anslen, beam_search_size, dropout_rate,
initial_learning_rate, mode='train'):
# inputs
self.inputs_q = tf.placeholder(tf.int32, shape=[batch_size, None], name='inputs_q') # index了
self.inputs_actual_length_q = tf.placeholder(tf.int32, [batch_size], name='inputs_actual_length_q')
self.inputs_p = tf.placeholder(tf.int32, shape=[batch_size, None], name='inputs_p')
self.inputs_actual_length_p = tf.placeholder(tf.int32, [batch_size], name='inputs_actual_length_p')
self.starts = tf.placeholder(tf.float32, [batch_size, None, 1]) # 论文后面说维度50是啥？？这里不应该就是一个数，即每个词0或1？
self.ends = tf.placeholder(tf.float32, [batch_size, None, 1])
# targets
if mode == 'train':
self.targets_a = tf.placeholder(tf.int32, shape=[batch_size, None], name='targets_a')
self.targets_actual_length_a = tf.placeholder(tf.int32, [batch_size], name='targets_actual_length_a')
# embeddings
# 如果要标点，则不能lookup，得自己设个词汇表embedding W，来更新；这里为方便先用下吧
with tf.variable_scope('embedding', reuse=tf.AUTO_REUSE):
embedding = tf.get_variable(name='embedding',
initializer=tf.truncated_normal(shape=[vocab_size, embedding_dim],
stddev=0.1))
# 截断的产生正态分布的函数，值与均值的差值大于两倍标准差则重新生成
inputs_embedded_q = tf.nn.embedding_lookup(embedding, self.inputs_q)
inputs_embedded_p = tf.nn.embedding_lookup(embedding, self.inputs_p)
if mode == 'train':
targets_embedded_a = tf.nn.embedding_lookup(embedding, self.targets_a)
inputs_concat_pos = tf.concat([self.starts, self.ends], axis=2)
inputs_concat_p = tf.concat([inputs_embedded_p, inputs_concat_pos], axis=2)
# print inputs_concat_pos, inputs_concat_p
# question encoder
with tf.variable_scope("q_encoder", reuse=tf.AUTO_REUSE):
fcell_q = tf.nn.rnn_cell.GRUCell(hidden_size)
bcell_q = tf.nn.rnn_cell.GRUCell(hidden_size)
fcell_q = tf.contrib.rnn.DropoutWrapper(fcell_q, output_keep_prob=1 - dropout_rate) # 有3个dropout，应该用哪个？？
bcell_q = tf.contrib.rnn.DropoutWrapper(bcell_q, output_keep_prob=1 - dropout_rate)
(fw_outputs_q, bw_outputs_q), (fw_final_state_q, bw_final_state_q) = \
tf.nn.bidirectional_dynamic_rnn(cell_fw=fcell_q,
cell_bw=bcell_q,
inputs=inputs_embedded_q,
sequence_length=self.inputs_actual_length_q,
dtype=tf.float32)
h_q = tf.concat((fw_outputs_q, bw_outputs_q), 2)
# print h_q # 输出是root，root_1的；但前面embedding是共享的
# print 'bw_outputs_q', bw_outputs_q
# passage encoder
with tf.variable_scope("p_encoder", reuse=tf.AUTO_REUSE):
fcell_p = tf.nn.rnn_cell.GRUCell(hidden_size)
bcell_p = tf.nn.rnn_cell.GRUCell(hidden_size)
fcell_p = tf.contrib.rnn.DropoutWrapper(fcell_p, output_keep_prob=1 - dropout_rate) # 有3个dropout，应该用哪个？？
bcell_p = tf.contrib.rnn.DropoutWrapper(bcell_p, output_keep_prob=1 - dropout_rate)
(fw_outputs_p, bw_outputs_p), (fw_final_state_p, bw_final_state_p) = \
tf.nn.bidirectional_dynamic_rnn(cell_fw=fcell_p,
cell_bw=bcell_p,
inputs=inputs_concat_p,
sequence_length=self.inputs_actual_length_p,
dtype=tf.float32)
h_p = tf.concat((fw_outputs_p, bw_outputs_p), 2)
# print h_p
# print 'bw_outputs_p', bw_outputs_p
# decoder
with tf.variable_scope("decoder", reuse=tf.AUTO_REUSE):
with tf.variable_scope("initial_state"):
w_d = tf.get_variable(name='w_d', shape=[hidden_size * 2, hidden_size * 2])
bias = tf.get_variable(name='bias', shape=[1, hidden_size * 2])
def compute_d_0(i, bw_p, bw_q, d_0_b):
bw_p_0_i = tf.reshape(bw_p[i][0], shape=[1, hidden_size]) # 第一个是这个0的吗
bw_q_0_i = tf.reshape(bw_q[i][0], shape=[1, hidden_size])
h_concat_i = tf.concat([bw_p_0_i, bw_q_0_i], axis=1)
d_0_i = tf.tanh(tf.matmul(h_concat_i, w_d) + bias) # d_0的维度也可随便定的，用矩阵w_d调即可；按论文，这里应该150吧
d_0_b.write(i, d_0_i)
i = tf.add(i, 1)
return i, bw_p, bw_q, d_0_b
d_0_b = tf.TensorArray(dtype=tf.float32, size=batch_size)
c = lambda x, y, z, w: tf.less(x, batch_size) # batch的循环
b = lambda x, y, z, w: compute_d_0(x, y, z, w)
i = tf.constant(0) # batch号
d_res = tf.while_loop(cond=c, body=b, loop_vars=(i, bw_outputs_p, bw_outputs_q, d_0_b))
d_0 = d_res[-1].stack()
# print 'd_0', d_0
with tf.variable_scope("attention_decoder_ouput", reuse=tf.AUTO_REUSE):
w_a = tf.get_variable(name='w_a', shape=[hidden_size * 2, hidden_size * 2])
u_a = tf.get_variable(name='u_a', shape=[hidden_size * 2, hidden_size * 2])
v_a = tf.get_variable(name='v_a', shape=[hidden_size * 2, 1])
cell_d = tf.nn.rnn_cell.GRUCell(hidden_size * 2) # 这个维度？？w,c,d怎么结合的；不对啊，d就是hidden_size大小吧
w_r = tf.get_variable(name='w_r', shape=[embedding_dim, hidden_size * 2]) # 注意要和词向量乘，维度不同
u_r = tf.get_variable(name='u_r', shape=[hidden_size * 2, hidden_size * 2])
v_r = tf.get_variable(name='v_r', shape=[hidden_size * 2, hidden_size * 2])
w_o = tf.get_variable(name='w_o', shape=[hidden_size, vocab_size]) # 是这里调吗？
def attention_step(h_p_i, h_q_i, d_t_1): # w_t应是当前输入词的embedding，也就是上一个输出的词
h = tf.concat([h_p_i, h_q_i], axis=0) # 不对啊词数都不同，这里怎么拼的？？直接第一维拼？嗯这里就按这种做法
# 感觉是这样的：拼是p+q，然后每个词做attention，所以还是一起乘，对的
# print d_t_1, h
sum_d = tf.matmul(d_t_1, w_a) + tf.matmul(h, u_a) # 矩阵了；h的行数是是p+q词数，列数是向量维度
# print 'h, sum_d', h, sum_d
s_t = tf.matmul(tf.tanh(sum_d), v_a) # 列向量
a_t = tf.nn.softmax(s_t)
a_t = tf.transpose(a_t) # 转成行向量
c_t = tf.matmul(a_t, h)
# print 'c_t', c_t
return c_t
def compute_m_t_j(j, r_t, m_t):
r_m = tf.maximum(r_t[0][j], r_t[0][j + 1])
# print r_m
m_t.write(j, r_m)
j = tf.add(j, 1)
return j, r_t, m_t
def maxout_hidden_layer(r_t):
m_t = tf.TensorArray(dtype=tf.float32, size=hidden_size)
c = lambda x, y, z: tf.less(x, hidden_size * 2 - 1)
b = lambda x, y, z: compute_m_t_j(x, y, z)
j = tf.constant(0)
m_t_res = tf.while_loop(cond=c, body=b, loop_vars=(j, r_t, m_t))
m_t = m_t_res[-1].stack()
m_t = tf.reshape(m_t, shape=[1, hidden_size])
# print 'm_t', m_t
return m_t
# train中两部分可分开；test中有依赖关系不可分开
if mode == 'train':
def output_step_train(w_t_1, c_t, d_t):
r_t = tf.matmul(w_t_1, w_r) + tf.matmul(c_t, u_r) + tf.matmul(d_t, v_r) # 其实w,c,d维度都不需一样， 用参数矩阵调即可；论文中说r_t是2d维，到底应该多少
# print 'r_t', r_t
m_t = maxout_hidden_layer(r_t) # maxout hidden layer
prob_t = tf.nn.softmax(tf.matmul(m_t, w_o)) # 行向量
# w_t_index = tf.argmax(prob_t, axis=1) # argmax??找出id，再embedding[id]
# print 'w_t_index', w_t_index
# w_t = tf.reshape(embedding[w_t_index[0]], shape=[1, -1])
return prob_t
def one_step_train(t, h_p_i, h_q_i, ta_i, d_t, w_t, prob_a): # 不对，train的时候w_t是输入的真实target的词
c_t = attention_step(h_p_i, h_q_i, d_t)
# print 'w_t, c_t, d_t ', w_t, c_t, d_t
w_c_d = tf.concat([w_t, c_t, d_t], axis=1)
# print 'w_c_d, d_t', w_c_d, d_t
out, d_tp1 = cell_d(inputs=w_c_d, state=d_t)
prob_t = output_step_train(w_t, c_t, d_tp1)
prob_a.write(t, prob_t)
# print 'prob_t', prob_t
# print ta_i[t]
w_tp1 = tf.reshape(ta_i[t], shape=[1, -1]) # -1是SOS，则从0开始是答案
t = tf.add(t, 1)
return t, h_p_i, h_q_i, ta_i, d_tp1, w_tp1, prob_a
def one_answer_train(i, prob_b):
c = lambda x, y, z, m, n, p, q: tf.less(x, self.targets_actual_length_a[i]) # 改这里，这里不应该是文章长，应该是答案长；应该根据生成词判断终止条件，也就是这里搞个函数，但回头再说，这里先简化
b = lambda x, y, z, m, n, p, q: one_step_train(x, y, z, m, n, p, q)
t = tf.constant(0) # batch号
h_p_i = h_p[i]
h_q_i = h_q[i]
d_0_i = d_0[i]
ta_i = targets_embedded_a[i]
# print 'd_0_i', d_0_i
w_0 = tf.reshape(embedding[SOS_ID], shape=[1, -1]) # 应是start标签的embedding；对的对的
prob_a = tf.TensorArray(dtype=tf.float32, size=self.targets_actual_length_a[i])
# print prob_a
res = tf.while_loop(cond=c, body=b, loop_vars=(t, h_p_i, h_q_i, ta_i, d_0_i, w_0, prob_a))
temp = tf.squeeze(res[-1].stack(), axis=1) # 具体做了什么，stack和squeeze的作用？
# print res[-1].stack(), temp
prob_b.write(i, temp)
i = tf.add(i, 1)
return i, prob_b
prob_b = tf.TensorArray(dtype=tf.float32, size=batch_size) # 应是batch，答案t长，词汇表长
c = lambda x, y: tf.less(x, batch_size) # batch的循环
b = lambda x, y: one_answer_train(x, y)
i = tf.constant(0) # batch号
prob_b_res = tf.while_loop(cond=c, body=b, loop_vars=(i, prob_b))
prob = prob_b_res[-1].stack()
# print 'prob', prob
if mode == 'test': # 要用beam search;先用最后只取概率最大的
def output_step_test(w_t_1, c_t, d_t, seq_w_ts_j, seq_w_ts_indices_j, seq_prob_ts_j):
# 这里需要考虑 输入！输入！有没有EOS之类的了
# 如果输入是结束符
# 就现在改了吧
def eos():
w_tp1s_j = tf.reshape(embedding[EOS_ID], shape=[1, -1]) # 就一直上EOS吧
seq_w_tp1s_j = tf.concat([seq_w_ts_j, w_tp1s_j], axis=0)
seq_w_tp1s_j = tf.expand_dims(seq_w_tp1s_j, 0)
w_tp1s_indices_j = tf.reshape([EOS_ID], shape=[1, 1])
seq_w_tp1s_indices_j = tf.expand_dims(seq_w_ts_indices_j, 0)
seq_w_tp1s_indices_j = tf.concat([seq_w_tp1s_indices_j, w_tp1s_indices_j], axis=1)
seq_prob_tp1s_j = tf.reshape(seq_prob_ts_j, shape=[1, 1])
# print seq_w_tp1s_j, seq_w_tp1s_indices_j, seq_prob_tp1s_j
return seq_w_tp1s_j, seq_w_tp1s_indices_j, seq_prob_tp1s_j
def not_eos():
r_t = tf.matmul(w_t_1, w_r) + tf.matmul(c_t, u_r) + tf.matmul(d_t, v_r)
# print 'r_t', r_t
m_t = maxout_hidden_layer(r_t) # maxout hidden layer
prob_t = tf.nn.softmax(tf.matmul(m_t, w_o)) # 行向量
# 先prob乘，选最大beam_size个
seq_prob_tp1s_all = seq_prob_ts_j[0] * prob_t # 应该是这么乘吧；这里取数
rs = tf.nn.top_k(seq_prob_tp1s_all, beam_search_size)
# 乘完的筛选差不多
# prob
seq_prob_tp1s = tf.reshape(rs.values, shape=[-1, 1]) # 直接替换，变列向量
# id
seq_w_tp1_indices = tf.concat([tf.reshape(seq_w_ts_indices_j, shape=[1, -1])]
* beam_search_size, axis=0) # 搞成行向量，重复12次，拼接
seq_w_tp1_indices = tf.concat([seq_w_tp1_indices, tf.transpose(rs.indices)], axis=1)
# embedding
seq_w_tp1s = tf.concat([tf.expand_dims(seq_w_ts_j, 0)] * beam_search_size,
axis=0) # 是不是有别的办法，这样写好奇怪
w_tp1s = tf.nn.embedding_lookup(embedding, rs.indices[0])
w_tp1s = tf.expand_dims(w_tp1s, 1) # 升维，第1维加
seq_w_tp1s = tf.concat([seq_w_tp1s, w_tp1s], axis=1)
# print 'seq_w_tp1s, seq_w_tp1_indices, seq_prob_tp1s', seq_w_tp1s, seq_w_tp1_indices, seq_prob_tp1s
return seq_w_tp1s, seq_w_tp1_indices, seq_prob_tp1s
result = tf.cond(tf.equal(seq_w_ts_indices_j[-1], tf.constant(EOS_ID)),
lambda: eos(), lambda: not_eos())
# print 'result', result
return result
def get_best_seqs(seq_w_ts, seq_w_ts_indices, seq_prob_ts):
# 可以转成字典，排序求最大，再取得相应下标？？
# 或连接，排序，再拆开
# 啊啊啊啊有现成函数
seq_prob_ts = tf.reshape(seq_prob_ts, shape=[1, -1])
rs = tf.nn.top_k(seq_prob_ts, beam_search_size) # 这里得是行向量
# print rs.indices
# 下面根据indices切片
best_seq_prob_ts = tf.reshape(rs.values, shape=[-1, 1]) # 这是行向量，看看改不改；改列向量吧
best_seq_w_ts = tf.gather(seq_w_ts, rs.indices[0]) # 都是对于第0维取
best_seq_w_ts_indices = tf.gather(seq_w_ts_indices, rs.indices[0])
# print 'best_seq_w_ts, best_seq_w_ts_indices, best_seq_prob_ts', \
# best_seq_w_ts, best_seq_w_ts_indices, best_seq_prob_ts
return best_seq_w_ts, best_seq_w_ts_indices, best_seq_prob_ts
def one_step_test(t, h_p_i, h_q_i, d_ts, best_seq_w_ts, best_seq_w_ts_indices, best_seq_prob_ts):
d_tp1s = []
seq_w_tp1s_list = []
seq_w_tp1_indices_list = []
seq_prob_tp1s_list = []
for j in range(beam_search_size): # 这里看看要不要也改成lambda；如果用lambda似乎要arraywrite了
seq_w_ts_j = best_seq_w_ts[j]
seq_w_ts_indices_j = best_seq_w_ts_indices[j]
seq_prob_ts_j = best_seq_prob_ts[j] # 先不取到数吧
# print 'seq_w_ts_j, seq_w_ts_indices_j, seq_prob_ts_j ', seq_w_ts_j, seq_w_ts_indices_j, seq_prob_ts_j
w_t = tf.reshape(seq_w_ts_j[-1], shape=[1, -1]) # 取最后一个？不对，返回的是全部啊；要么在外面拼，要么在里面拼
d_t = tf.reshape(d_ts[j], shape=[1, -1]) # 这也复数
c_t = attention_step(h_p_i, h_q_i, d_t)
w_c_d = tf.concat([w_t, c_t, d_t], axis=1)
# print 'w_t, c_t, d_t, w_c_d ', w_t, c_t, d_t, w_c_d
out, d_tp1 = cell_d(inputs=w_c_d, state=d_t) # 但是连d都不一样，这咋搞
d_tp1s.append(d_tp1)
seq_w_tp1s, seq_w_tp1_indices, seq_prob_tp1s = \
output_step_test(w_t, c_t, d_tp1, seq_w_ts_j, seq_w_ts_indices_j, seq_prob_ts_j)
# 输出得12*12后再一起筛选成12；具体咋筛呢；还要这里w不能只是当前词了，得是从1至当前序列的所有词
# 每个12个就够了，自己都进不了前12的，其他也进不了
seq_w_tp1s_list.append(seq_w_tp1s)
seq_w_tp1_indices_list.append(seq_w_tp1_indices)
seq_prob_tp1s_list.append(seq_prob_tp1s)
# 拼起来
d_tp1s = tf.concat(d_tp1s, axis=0)
seq_w_tp1s = tf.concat(seq_w_tp1s_list, axis=0)
seq_w_tp1_indices = tf.concat(seq_w_tp1_indices_list, axis=0)
seq_prob_tp1s = tf.concat(seq_prob_tp1s_list, axis=0) # 看如果是列向量就是这样
# 筛选
best_seq_w_tp1s, best_seq_w_tp1_indices, best_seq_prob_tp1s = \
get_best_seqs(seq_w_tp1s, seq_w_tp1_indices, seq_prob_tp1s)
# 求下not_all_eos；判断最后是不是EOS都
# ???不会求啊。。。。先不管他，直接搞到最后好了
t =tf.add(t, 1)
return t, h_p_i, h_q_i, d_tp1s, best_seq_w_tp1s, best_seq_w_tp1_indices, best_seq_prob_tp1s
# 每次应返回到目前为止概率最大的beam_size个序列（ids，embeddings）及各自概率
# 第一步就是一个SOS，但后面是beam_size的输入了，但也有可能有重复的词；t=0在函数里分布对待好了
def output_step_test_t_0(w_0, c_0, d_1):
r_0 = tf.matmul(w_0, w_r) + tf.matmul(c_0, u_r) + tf.matmul(d_1, v_r)
# 其实w,c,d维度都不需一样， 用参数矩阵调即可；论文中说r_t是2d维，到底应该多少
# print 'r_0', r_0
m_t = maxout_hidden_layer(r_0) # maxout hidden layer
prob_0 = tf.nn.softmax(tf.matmul(m_t, w_o)) # 行向量
rs = tf.nn.top_k(prob_0, beam_search_size)
# prob
seq_prob_1s = tf.reshape(rs.values, shape=[-1, 1])
# print 'seq_prob_1s', seq_prob_1s
# id
seq_w_0_indices = tf.reshape(tf.constant([SOS_ID] * beam_search_size), shape=[-1, 1]) # 搞成列向量
# print 'seq_w_0_indices', seq_w_0_indices
seq_w_1_indices = tf.concat([seq_w_0_indices, tf.transpose(rs.indices)], axis=1) # e二维的
# embedding
seq_w_0s = tf.concat([w_0] * beam_search_size, axis=0)
seq_w_0s = tf.expand_dims(seq_w_0s, 1) # 升维，第1维加
w_1s = tf.nn.embedding_lookup(embedding, rs.indices[0])
w_1s = tf.expand_dims(w_1s, 1) # 升维，第1维加
seq_w_1s = tf.concat([seq_w_0s, w_1s], axis=1) # SOS 拼上相应词
# print 'seq_w_1', seq_w_1s
return seq_w_1s, seq_w_1_indices, seq_prob_1s
def one_step_test_t_0(w_0, d_0, h_p_i, h_q_i):
c_0 = attention_step(h_p_i, h_q_i, d_0)
w_c_d = tf.concat([w_0, c_0, d_0], axis=1)
# print 't==0 ', w_0, c_0, d_0, w_c_d
out, d_1 = cell_d(inputs=w_c_d, state=d_0)
best_seq_w_1s, best_seq_w_1_indices, best_seq_prob_1s \
= output_step_test_t_0(w_0, c_0, d_1)
d_1s = tf.concat([d_1] * beam_search_size, axis=0) # 重复下；行向量拼的
return d_1s, best_seq_w_1s, best_seq_w_1_indices, best_seq_prob_1s
def stop_condition(t, best_seq_w_t_indices): # 应是到最大长度了，或前beam_size个概率最大的序列全到结束符了
w_t_indices = best_seq_w_t_indices[:, -1] # 先抽出每个序列最后一个词的indices，直接切片即可吧
all_eos = tf.constant([EOS_ID]*beam_search_size) # 拼出EOS的
cond1 = tf.equal(w_t_indices, all_eos) # 然后equals进行比较
cond1 = tf.logical_not(tf.reduce_all(cond1)) # True不终止，False终止
cond2 = t < max_anslen # True不终止，False终止
cond = tf.logical_and(cond1, cond2) # 都True才不终止，一个False即终止
# print cond
return cond
def one_answer(i, a_ids_b):
# t=0时单独搞一个
h_p_i = h_p[i]
h_q_i = h_q[i]
d_0_i = d_0[i]
w_0 = tf.reshape(embedding[SOS_ID], shape=[1, -1])
d_1s, best_seq_w_1s, best_seq_w_1_indices, best_seq_prob_1s = one_step_test_t_0(w_0, d_0_i, h_p_i, h_q_i)
# t=1开始
t = tf.constant(1)
c = lambda x, y, z, m, n, p, q: stop_condition(x, p)
b = lambda x, y, z, m, n, p, q: one_step_test(x, y, z, m, n, p, q)
res = tf.while_loop(cond=c, body=b, loop_vars=[t, h_p_i, h_q_i, d_1s,
best_seq_w_1s, best_seq_w_1_indices,
best_seq_prob_1s],
shape_invariants=[t.get_shape(), h_p_i.get_shape(),
h_q_i.get_shape(), d_1s.get_shape(),
tf.TensorShape([beam_search_size, None, embedding_dim]),
tf.TensorShape([beam_search_size, None]),
tf.TensorShape([beam_search_size, 1])])
# 可能还有问题，后面不加词了是不是需要padding；应该要，里面维度要一样
# 到最后一步取最大概率的序列，填入a_ids_b
seq_probs = res[-1]
# w_ts = res[-3].stack()
w_ts_indices = res[-2]
best_seq_no = tf.argmax(seq_probs)
best_seq_no = best_seq_no[0]
# best_w_ts = w_ts[best_seq_no]
best_a_ids = w_ts_indices[best_seq_no]
print 'best_seq_no, best_a_ids ', best_seq_no, best_a_ids
a_ids_b.write(i, best_a_ids)
i = tf.add(i, 1)
return i, a_ids_b
answers_ids_b = tf.TensorArray(dtype=tf.int32, size=batch_size)
c = lambda x, y: tf.less(x, batch_size) # batch的循环
b = lambda x, y: one_answer(x, y)
i = tf.constant(0) # batch号
answers_ids_b_res = tf.while_loop(cond=c, body=b, loop_vars=(i, answers_ids_b))
# print 'prob', prob
self.answers_ids = answers_ids_b_res[-1].stack()
print 'answers_ids', self.answers_ids
# loss, train_op
if mode == 'train':
with tf.variable_scope('loss', reuse=tf.AUTO_REUSE):
def compute_loss_step(t, ta_index_i, prob_i, loss_a):
ta_index_t = ta_index_i[t]
prob_t = prob_i[t][ta_index_t]
# print prob_t
loss_a *= prob_t
t = tf.add(t, 1)
return t, ta_index_i, prob_i, loss_a
def compute_loss_batch(i, loss):
t = tf.constant(0)
prob_i = prob[i]
ta_index_i = self.targets_a[i]
loss_a = tf.constant(1.0)
c = lambda x, y, z, m: tf.less(x, self.targets_actual_length_a[i])
b = lambda x, y, z, m: compute_loss_step(x, y, z, m)
loss_res = tf.while_loop(cond=c, body=b, loop_vars=(t, ta_index_i, prob_i, loss_a))
loss_a = loss_res[-1]
loss += loss_a
# print loss_a, loss
i = tf.add(i, 1)
return i, loss
loss = tf.constant(0.0)
c = lambda x, y: tf.less(x, batch_size)
b = lambda x, y: compute_loss_batch(x, y)
loss_res = tf.while_loop(cond=c, body=b, loop_vars=(i, loss))
self.loss = loss_res[-1]
# print 'loss', self.loss
with tf.variable_scope('train_op', reuse=tf.AUTO_REUSE):
tvars = tf.trainable_variables()
# print tvars
# grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars), grad_clip)
optimizer = tf.train.AdamOptimizer(initial_learning_rate)
grads_and_vars = optimizer.compute_gradients(self.loss)
self.train_op = optimizer.apply_gradients(grads_and_vars) # 还是报一样的错
# self.train_op = optimizer.apply_gradients(zip(grads, tvars))
print self.train_op
def swap_strongest(colonies, empires, empires_numbers, num_of_colonies,
cost_function):
colonies_power = helpers.evaluate_countries_power(colonies, cost_function)
colonies_indexes = helpers.create_index_column(colonies_power)
colonies_power_with_index = helpers.concatenate_tensor_with_index(
colonies_power, colonies_indexes, num_of_colonies)
empires_power = helpers.evaluate_countries_power(empires, cost_function)
empires_numbers_to_check, _ = tf.unique(empires_numbers)
swap_initial_params = (constants.int_zero, empires_numbers_to_check,
colonies, empires, empires_numbers, empires_power,
colonies_power, tf.squeeze(colonies_indexes),
colonies_power_with_index)
_, _, new_colonies, new_empires, _, new_empires_power, new_colonies_power, _, _ = tf.while_loop(
condition_swap_strongest, body_swap_strongest, swap_initial_params)
return new_colonies, new_empires, new_empires_power, new_colonies_power
def constrained_l2_proj_distance_save_memory(x, grads, biases):
u, w, b = x, grads, -biases[:, :, None]
ndim = K.int_shape(u)[1]
neurons = K.int_shape(w)[1]
w = K.reshape(w, (-1, ndim))
b = K.reshape(b, (-1, 1))
# Handy naming to make code a bit more readable.
def is_(x):
return K.cast(x, 'float32')
def where(x):
return K.cast(x, 'float32')
# Check feasibility.
# The problem is infeasible if `q` (defined below) can be brought to
# infinity as `lam` -> +/-infinity. We see that as `lam` -> +infinity,
# `x_star[i]` becomes 0 if `w[i]` is positive and if `w[i]` is negative.
# Thus, dq/dlam = -b + sum_{i : w_i < 0}{w_i}. If dq/dlam is positive, then
# `q` will go to infinity. The other case comes from the symmetric case for
# when `lam` -> -infinity.
infeasible = (is_(K.sum(w * where(w > 0), axis=1) < b[:, 0]) +
is_(K.sum(w * where(w < 0), axis=1) > b[:, 0]))
feasible = 1. - infeasible
# Get the order (as lambda goes from -infinity to 0) in which each dimention
# transitions to the I stage.
I_in_order = tf.argsort(u / w * where(w < 0) + (u - 1) / w * where(w > 0))
# Get the order (as lambda goes from 0 to +infinity) in which each dimention
# transitions out of the I stage.
I_out_order = tf.argsort(u / w * where(w > 0) + (u - 1) / w * where(w < 0))
w_I_in = tf.gather(w, I_in_order, batch_dims=1)
u_I_in = tf.gather(u, I_in_order, batch_dims=1)
w_1_in = tf.gather(w * where(w > 0), I_in_order, batch_dims=1)
in_nums = w_I_in * u_I_in - w_1_in
in_denoms = w_I_in**2
w_I_out = tf.gather(w, I_out_order, batch_dims=1)
u_I_out = tf.gather(u, I_out_order, batch_dims=1)
w_1_out = tf.gather(w * where(w < 0), I_out_order, batch_dims=1)
out_nums = -w_I_out * u_I_out + w_1_in
out_denoms = -w_I_out**2
nums = (K.sum(w * where(w > 0), axis=1)[:, None] +
K.cumsum(K.concatenate(
(in_nums, out_nums), axis=1), axis=1)[:, :-1] - b)
denoms = K.cumsum(K.concatenate((in_denoms, out_denoms), axis=1),
axis=1)[:, :-1]
argmaxes = nums / denoms
# Find the inflection points in `q`.
inflections = K.concatenate(((u - 1) / w, u / w))
lam = K.concatenate((argmaxes, inflections))
i0 = tf.constant(0)
m0 = K.zeros((0, ndim))
loop_condition = lambda i, m: i < K.shape(lam)[0]
def loop_body(i, m):
x_star_candidate_i = K.clip(u[0, None] - lam[i, :, None] * w[i, None],
0., 1.)
max_q_candidate_i = (
K.sum(.5 * (x_star_candidate_i - u[0, None])**2 +
lam[i, :, None] * w[i, None] * x_star_candidate_i,
axis=-1) - lam[i] * b[i])
opt_i = K.cast(K.argmax(max_q_candidate_i, axis=0), 'int32')
return [i + 1, tf.concat((m, x_star_candidate_i[opt_i, None]), axis=0)]
x_star = tf.while_loop(
loop_condition,
loop_body,
loop_vars=[i0, m0],
shape_invariants=[i0.get_shape(),
tf.TensorShape([None, ndim])])[1]
d = tf.norm(x_star - u, axis=1) / feasible
return K.reshape(d, (-1, neurons))
def inference(data, sequence_length, keep_prob):
"""
The inference function.
:param data: [batch_size, 107, 20]
:param sequence_length: [batch_size]
:param keep_prob: the parameter for dropout layer.
:return: the logits.
"""
tf.set_random_seed(0)
batch_size_op = tf.shape(data)[0]
max_length_op = tf.shape(data)[1]
with tf.variable_scope('lstm_variable_sequence') as scope:
# For the LSTM weight and biases initialization.
cell_fw = tf.nn.rnn_cell.LSTMCell(
num_units=64, initializer=tf.glorot_normal_initializer(seed=0))
cell_fw = tf.nn.rnn_cell.DropoutWrapper(cell_fw,
output_keep_prob=keep_prob,
seed=0)
cell_bw = tf.nn.rnn_cell.LSTMCell(
num_units=64, initializer=tf.glorot_normal_initializer(seed=0))
cell_bw = tf.nn.rnn_cell.DropoutWrapper(cell_bw,
output_keep_prob=keep_prob,
seed=0)
init_fw = cell_fw.zero_state(batch_size_op, dtype=tf.float32)
init_bw = cell_bw.zero_state(batch_size_op, dtype=tf.float32)
bidrnn_outputs, final_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=data,
sequence_length=sequence_length,
initial_state_fw=init_fw,
initial_state_bw=init_bw)
fw_lstm_outputs = final_states[0][1]
bw_lstm_outputs = final_states[1][1]
lstm_outputs = tf.concat((fw_lstm_outputs, bw_lstm_outputs),
axis=1) # shape = [batch_size, 128]
with tf.variable_scope('conv_pssm') as scope:
matrix = sio.loadmat('./program/model_dir/pssm.mat')['pssm']
initializer_filters = tf.reshape(tf.constant(matrix, dtype=tf.float32),
[1, 20, 1, 20])
initializer_biases = tf.constant_initializer(0)
filters = tf.get_variable('filters',
initializer=initializer_filters,
dtype=tf.float32)
biases = tf.get_variable('biases', [20],
initializer=initializer_biases,
dtype=tf.float32,
trainable=False)
input = tf.reshape(data, [batch_size_op, max_length_op, 20, 1])
temp = tf.nn.conv2d(input,
filters,
strides=[1, 1, 20, 1],
padding='SAME')
temp_b = tf.nn.bias_add(temp, biases)
conv_pssm = temp_b # shape= [batch_size, 107, 1, 20]
bandwidth = tf.floor(tf.divide(sequence_length, 4))
width = tf.cast(tf.multiply(bandwidth, 4), tf.int32)
Tensor_array = tf.TensorArray(tf.float32, batch_size_op)
def cond(i, array):
return i < batch_size_op
def body(i, array):
avblock_temp = tf.reshape(conv_pssm[i][0:width[i]], [4, -1, 20])
avblock = tf.reshape(tf.reduce_mean(avblock_temp, axis=1), [4, 20])
array = array.write(i, avblock)
return i + 1, array
i, array = tf.while_loop(cond, body, (0, Tensor_array))
outputs = array.stack()
with tf.variable_scope('conv_feature_extraction') as scope:
initializer_filters = tf.truncated_normal_initializer(stddev=0.4,
seed=0)
initializer_biases = tf.constant_initializer(0)
filters = tf.get_variable('filters', [4, 4, 1, 20],
initializer=initializer_filters,
dtype=tf.float32)
biases = tf.get_variable('biases', [20],
initializer=initializer_biases,
dtype=tf.float32)
input = tf.reshape(outputs, [batch_size_op, 4, 20, 1])
temp = tf.nn.conv2d(input,
filters,
strides=[1, 4, 4, 1],
padding='SAME')
temp_b = tf.nn.bias_add(temp, biases)
conv_feature_extraction = tf.nn.relu(
temp_b) # shape= [batch_size, 1, 5, 20]
with tf.variable_scope('dropout') as scope:
dropout = tf.nn.dropout(conv_feature_extraction,
keep_prob=keep_prob,
seed=0)
with tf.variable_scope('Merge_features') as scope:
conv = tf.reshape(dropout, [batch_size_op, 100])
merge_features = tf.concat([lstm_outputs, conv],
axis=1) # shape = [batch_size, 228]
with tf.variable_scope('fully_connected_1') as scope:
initializer_weights = tf.truncated_normal_initializer(stddev=0.4,
seed=0)
initializer_biases = tf.constant_initializer(0.1)
weights = tf.get_variable('weight', [228, 100],
initializer=initializer_weights,
dtype=tf.float32)
biases = tf.get_variable('biases', [100],
initializer=initializer_biases,
dtype=tf.float32)
f1_l2_loss = tf.multiply(tf.nn.l2_loss(weights),
0.2,
name='f1_weight_loss')
tf.add_to_collection('losses', f1_l2_loss)
temp = tf.nn.xw_plus_b(merge_features, weights, biases)
fc1 = tf.nn.relu(temp)
with tf.variable_scope('fully_connected_2') as scope:
initializer_weights = tf.truncated_normal_initializer(stddev=0.4,
seed=0)
initializer_biases = tf.constant_initializer(0.1)
weights = tf.get_variable('weight', [100, 2],
initializer=initializer_weights,
dtype=tf.float32)
biases = tf.get_variable('biases', [2],
initializer=initializer_biases,
dtype=tf.float32)
f2_l2_loss = tf.multiply(tf.nn.l2_loss(weights),
0.2,
name='f2_weight_loss')
tf.add_to_collection('losses', f2_l2_loss)
logits = tf.nn.xw_plus_b(fc1, weights, biases)
return logits
def yolo_loss(args,
anchors,
ignore_thresh=.5,
seg_loss_weight=0.1,
print_loss=False):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:1]
mask_prob = args[1]
co_enegy = args[2]
y_true = args[3:4]
mask_gt = args[4]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [
[0, 1, 2]
] ##due to deleting 2 scales change [[6,7,8], [3,4,5], [0,1,2]] to [[0,1,2]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32,
K.dtype(y_true[0])) # x32 is original size
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
] #3 degree scales output
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
# true_class_probs = y_true[l][..., 5:] #... ==????
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
def smooth_L1(y_true, y_pred, sigma=3.0):
""" Create a smooth L1 loss functor.
Args
sigma: This argument defines the point where the loss changes from L2 to L1.
Returns
A functor for computing the smooth L1 loss given target data and predicted data.
"""
sigma_squared = sigma**2
# compute smooth L1 loss
# f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma
# |x| - 0.5 / sigma / sigma otherwise
regression_diff = y_true - y_pred
regression_diff = K.abs(regression_diff)
regression_loss = tf.where(
K.less(regression_diff, 1.0 / sigma_squared),
0.5 * sigma_squared * K.pow(regression_diff, 2),
regression_diff - 0.5 / sigma_squared)
return regression_loss
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * smooth_L1(
raw_true_wh, raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
(1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
seg_loss = K.binary_crossentropy(mask_gt, mask_prob, from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
seg_loss = K.sum(seg_loss) / mf
co_enegy_loss = cem_loss(co_enegy) / mf
loss += xy_loss + wh_loss + confidence_loss + seg_loss * seg_loss_weight + co_enegy_loss
if print_loss:
loss = tf.Print(loss, [
'\n'
'co_peak_loss: ', co_enegy_loss, 'co_peak_energe: ',
K.sum(co_enegy) / mf
],
message='loss: ')
return K.expand_dims(loss, axis=0)
def map_fn(x):
"""Internal function to flat_map over.
Consumes a batch of input examples and produces a variable number of output
examples.
Args:
x: a single example
Returns:
a tf.data.Dataset
"""
partial = empty_example.copy()
i = tf.zeros([], dtype=tf.int32)
dynamic_batch_size = tf.shape(x[keys[0]])[0]
outputs = {}
for k in keys:
outputs[k] = tf.TensorArray(
tf.int32, size=0, dynamic_size=True, element_shape=[length])
outputs[k + "_position"] = tf.TensorArray(
tf.int32, size=0, dynamic_size=True, element_shape=[length])
def cond_fn(i, partial, outputs):
del partial, outputs
return i < dynamic_batch_size
def body_fn(i, partial, outputs):
"""Body function for while_loop.
Args:
i: integer scalar
partial: dictionary of Tensor (partially-constructed example)
outputs: dictionary of TensorArray
Returns:
A triple containing the new values of the inputs.
"""
can_append = True
one_example = {}
for k in keys:
val = tf.cast(x[k][i], tf.int32)
val = val[:tf.reduce_sum(tf.cast(tf.not_equal(val, 0), tf.int32))]
one_example[k] = val
for k in keys:
can_append = tf.logical_and(
can_append,
tf.less_equal(
tf.size(partial[k]) + tf.size(one_example[k]), length))
def false_fn():
return write_packed_example(partial, outputs)
def true_fn():
return partial, outputs
partial, outputs = tf.cond(can_append, true_fn, false_fn)
new_partial = {}
for k in keys:
new_seq = one_example[k][:length]
new_seq_len = tf.size(new_seq)
new_partial[k] = tf.concat([partial[k], new_seq], 0)
new_partial[k + "_position"] = tf.concat(
[partial[k + "_position"],
tf.range(new_seq_len, dtype=tf.int32)], 0)
partial = new_partial
return i+1, partial, outputs
i, partial, outputs = tf.while_loop(
cond_fn, body_fn, (i, partial, outputs),
back_prop=False,
shape_invariants=(
tf.TensorShape([]),
{k: tf.TensorShape([None]) for k in keys_etc},
{k: tf.TensorShape(None) for k in keys_etc},
))
partial, outputs = write_packed_example(partial, outputs)
packed = {k: outputs[k].stack() for k in keys_etc}
for k in keys:
packed[k + "_segmentation"] = (
tf.cumsum(
tf.cast(tf.equal(packed[k + "_position"], 0), tf.int32), axis=1) *
tf.cast(tf.not_equal(packed[k], 0), tf.int32))
return packed
def sample_sequence(*,
hparams,
length,
start_token=None,
batch_size=None,
context=None,
sampler='k',
temperature=1,
top_k=0,
alpha=0.05,
nuc_prob=0.25,
flat_prob=0.02,
k_window_size=None,
window_weights=None):
if start_token is None:
assert context is not None, 'Specify exactly one of start_token and context!' # this is where the whole context is already given into the model.
# it is the primer that I write for it!
else:
assert context is None, 'Specify exactly one of start_token and context!'
context = tf.fill([batch_size, 1],
start_token) # this is not used in my case!
def step(hparams, tokens, past=None):
lm_output = model.model(hparams=hparams,
X=tokens,
past=past,
reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(
model.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
with tf.name_scope('sample_sequence'):
# this will store all of the logits for a specific sampling run.
# ultimately I want this to be a: samples * prompts (batch size) * words * length sized matrix.
#all_logits = tf.Variable(tf.zeros([batch_size, 50257, length]), dtype=tf.float32)
#all_logits = tf.Variable(name='all_logits', shape=[batch_size, 50257, length], initializer= ,dtype=tf.float32, trainable=False)
def body(past, prev, output, all_logits):
next_outputs = step(hparams, prev, past=past)
logits = next_outputs['logits'][:,
-1, :] / tf.to_float(temperature)
if sampler == 'k':
print('using top k')
logits = top_k_logits(logits, k=top_k)
elif sampler == 'n':
print('using nucleus')
logits = nucleus(logits, p=nuc_prob)
elif sampler == 'tfs':
print('using tail free sampling')
logits = tail_free(logits,
alpha) #, k_window_size, window_weights)
elif sampler == 'flat':
print('using flat percentage sampling')
logits = flat_perc(logits, flat_prob)
else:
print('defauling to top k sampling')
logits = top_k_logits(logits, k=top_k)
#print('the logits shape post processing is: ', logits.shape)
samples = tf.multinomial(logits,
num_samples=1,
output_dtype=tf.int32)
#print('the samples shape is: ', samples.shape)
return [
next_outputs['presents'] if past is None else tf.concat(
[past, next_outputs['presents']], axis=-2),
tf.reshape(samples, [batch_size, 1]),
tf.concat([output, samples], axis=1),
tf.expand_dims(next_outputs['logits'][:, -1, :], axis=2)
if all_logits is None else tf.concat([
all_logits,
tf.expand_dims(next_outputs['logits'][:, -1, :], axis=2)
],
axis=2)
#tf.concat([all_logits, tf.expand_dims(next_outputs['logits'][:, -1, :], axis=2)], axis=2)
#all_logits[:,:,tf.shape(output)[1]+1].assign(tf.expand_dims(next_outputs['logits'][:, -1, :], axis=2) )
]
past, prev, output, all_logits = body(
None, context, context, None
) # for the first run the output and previous are both the context.
def cond(*args):
return True
_, _, tokens, all_logits_out = tf.while_loop(
cond=cond,
body=body,
maximum_iterations=length - 1,
loop_vars=[past, prev, output, all_logits],
#changed the 2nd shape invariant so that it can handle the ? shape (which is actually batch size) for the TFS sampling.
shape_invariants=[
tf.TensorShape(
model.past_shape(hparams=hparams, batch_size=batch_size)),
tf.TensorShape([batch_size, None]),
tf.TensorShape([batch_size, None]),
tf.TensorShape([batch_size, 50257, None]) #batch size
],
back_prop=False,
)
return (tokens, all_logits_out)
def __init__(self,
inp,
inp_mask,
inp_id,
seq2seq_gtruth,
post_gtruth,
hyper_params=None,
training=True,
name='Tacotron',
reuse=False):
"""
Build the computational graph.
:param inp:
:param inp_mask:
:param seq2seq_gtruth:
:param post_gtruth:
:param hyper_params:
:param training:
:param name:
"""
super(Tacotron, self).__init__(name)
self.hyper_params = HyperParams(
) if hyper_params is None else hyper_params
with tf.variable_scope(name, reuse=reuse):
self.global_step = tf.Variable(0,
name='global_step',
trainable=False)
self.learning_rate = tf.Variable(
self.hyper_params.learning_rate[0],
name='learning_rate',
trainable=False,
dtype=tf.float32)
batch_size = tf.shape(inp)[0]
input_time_steps = tf.shape(inp)[1]
output_time_steps = tf.shape(seq2seq_gtruth)[1]
### Encoder [begin]
with tf.variable_scope('character_embedding'):
embed_inp = EmbeddingLayer(self.hyper_params.embed_class,
self.hyper_params.embed_dim)(inp)
with tf.variable_scope("changeToVarible"):
self.single_style_token = tf.get_variable(
'style_token', (1, self.hyper_params.styles_kind,
self.hyper_params.style_dim),
dtype=tf.float32)
# self.Popt = tf.Print(self.single_style_token, [self.single_style_token], message='style:')
self.style_token = tf.tile(self.single_style_token,
(batch_size, 1, 1))
with tf.variable_scope("inp_att_onehot"):
inp_att = tf.one_hot(inp_id,
depth=self.hyper_params.styles_kind,
dtype=tf.float32)
self.inp_att = inp_att
# inp_att = tf.Print(inp_att, [inp_att], message='input_att:', summarize=10)
# print(inp_att)
with tf.variable_scope('encoder_pre_net'):
pre_ed_inp = tf.layers.dropout(tf.layers.dense(
embed_inp, 256, tf.nn.relu),
training=training)
pre_ed_inp = tf.layers.dropout(tf.layers.dense(
pre_ed_inp, 128, tf.nn.relu),
training=training)
encoder_output = modules.cbhg(pre_ed_inp,
training=training,
k=16,
bank_filters=128,
projection_filters=(128, 128),
highway_layers=4,
highway_units=128,
bi_gru_units=128,
sequence_length=inp_mask,
name='encoder_cbhg',
reuse=False)
# with tf.variable_scope('post_text'):
# all_outputs, _ = tf.nn.dynamic_rnn(cell=GRUCell(256), inputs=encoder_output, sequence_length=inp_mask,
# dtype=encoder_output.dtype, parallel_iterations=unkonwn_parallel_iterations)
# all_outputs = tf.transpose(all_outputs, [1, 0, 2])
# static_encoder_output = all_outputs[-1]
# ### Encoder [end]
#
# sentence_style_att = tf.layers.dense(static_encoder_output, 256, tf.nn.relu)
# sentence_style_att = tf.layers.dense(sentence_style_att, 64, tf.nn.relu)
# sentence_style = tf.layers.dense(sentence_style_att, 10, tf.nn.softmax)
# sentence_style = tf.Print(sentence_style, [sentence_style[0]], message='sentence', summarize=10)
sentence_style = tf.reduce_sum(tf.expand_dims(inp_att, axis=-1) *
self.style_token,
axis=1)
# sentence_style = tf.Print(sentence_style, [sentence_style], message='sentence_style', summarize=10)
### Attention Module
with tf.variable_scope('attention'):
att_module = AttentionModule(256,
encoder_output,
sequence_length=inp_mask,
time_major=False)
# with tf.variable_scope("attention_style"):
# att_module_style = AttentionModule(256, self.style_token, time_major=False)
### Decoder [begin]
att_cell = GRUCell(256)
dec_cell = MultiRNNCell(
[ResidualWrapper(GRUCell(256)) for _ in range(2)])
# prepare output alpha TensorArray
with tf.variable_scope('prepare_decode'):
reduc = self.hyper_params.reduction_rate
reduced_time_steps = tf.div(output_time_steps, reduc)
init_att_cell_state = att_cell.zero_state(
batch_size, tf.float32)
init_dec_cell_state = dec_cell.zero_state(
batch_size, tf.float32)
init_state_tup = tuple(
[init_att_cell_state, init_dec_cell_state])
init_output_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
init_alpha_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
init_weight_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
init_weight_per_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
# init_alpha_style_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
time_major_seq2seq_gtruth = tf.transpose(seq2seq_gtruth,
perm=(1, 0, 2))
indic_array = tf.concat([
tf.zeros([
reduc, batch_size, self.hyper_params.seq2seq_dim
]), time_major_seq2seq_gtruth
],
axis=0)
init_context = tf.zeros([batch_size, 256], dtype=tf.float32)
# init_context_style = tf.zeros([batch_size, 256], dtype=tf.float32)
init_time = tf.constant(0, dtype=tf.int32)
cond = lambda x, *_: tf.less(x, reduced_time_steps)
def body(this_time, old_context, old_output_ta, old_alpha_ta,
old_weight_per_ta, old_state_tup):
with tf.variable_scope('decoder_pre_net'):
dec_pre_ed_inp = indic_array[reduc * this_time + reduc - 1]
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(
dec_pre_ed_inp, 256, tf.nn.relu),
training=training)
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(
dec_pre_ed_inp, 128, tf.nn.relu),
training=training)
with tf.variable_scope('attention_rnn'):
att_cell_inp = tf.concat([old_context, dec_pre_ed_inp],
axis=-1)
att_cell_out, att_cell_state = att_cell(
att_cell_inp, old_state_tup[0])
with tf.variable_scope('attention'):
query = att_cell_state[0]
context, alpha = att_module(query)
new_alpha_ta = old_alpha_ta.write(this_time, alpha)
with tf.variable_scope('decoder_rnn'):
# context = tf.Print(context, [context], message='context', summarize=10)
weighting_context = context + sentence_style
weight_per = tf.reduce_mean(
tf.abs(sentence_style[0]) /
(tf.abs(context[0]) + tf.abs(sentence_style[0])))
# weight_per = tf.Print(weight_per, [weight_per], message='weight', summarize=10)
new_weight_per_ta = old_weight_per_ta.write(
this_time, weight_per)
dec_input = tf.layers.dense(
tf.concat([att_cell_out, weighting_context], axis=-1),
256)
dec_cell_out, dec_cell_state = dec_cell(
dec_input, old_state_tup[1])
dense_out = tf.layers.dense(
dec_cell_out, self.hyper_params.seq2seq_dim * reduc)
new_output_ta = old_output_ta.write(this_time, dense_out)
new_state_tup = tuple([att_cell_state, dec_cell_state])
return tf.add(
this_time, 1
), context, new_output_ta, new_alpha_ta, new_weight_per_ta, new_state_tup
# run loop
_, _, seq2seq_output_ta, alpha_ta, weight_per_ta, *_ = tf.while_loop(
cond,
body, [
init_time, init_context, init_output_ta, init_alpha_ta,
init_weight_per_ta, init_state_tup
],
parallel_iterations=unkonwn_parallel_iterations)
with tf.variable_scope('reshape_decode'):
seq2seq_output = tf.reshape(
seq2seq_output_ta.stack(),
shape=(reduced_time_steps, batch_size,
self.hyper_params.seq2seq_dim * reduc))
seq2seq_output = tf.reshape(
tf.transpose(seq2seq_output, perm=(1, 0, 2)),
shape=(batch_size, output_time_steps,
self.hyper_params.seq2seq_dim))
self.seq2seq_output = seq2seq_output
alpha_output = tf.reshape(alpha_ta.stack(),
shape=(reduced_time_steps,
batch_size, input_time_steps))
alpha_output = tf.expand_dims(
tf.transpose(alpha_output, perm=(1, 0, 2)), -1)
self.alpha_output = alpha_output
alpha_style_att = inp_att
alpha_style_att = tf.expand_dims(alpha_style_att,
-1) # batch major
alpha_style_att = tf.expand_dims(alpha_style_att,
-1) # batch major
alpha_style_att = tf.transpose(alpha_style_att,
perm=(0, 2, 1, 3))
self.alpha_style_att = alpha_style_att
weight_per_ta = tf.reshape(weight_per_ta.stack(),
shape=(reduced_time_steps, 1))
self.weight_per_ta = weight_per_ta
### Decoder [end]
### PostNet [begin]
post_output = modules.cbhg(
seq2seq_output,
training=training,
k=8,
bank_filters=128,
projection_filters=(256, self.hyper_params.seq2seq_dim),
highway_layers=4,
highway_units=128,
bi_gru_units=128,
sequence_length=None,
name='decoder_cbhg',
reuse=False)
post_output = tf.layers.dense(post_output,
self.hyper_params.post_dim,
name='post_linear_transform')
self.post_output = post_output
### PostNet [end]
### Loss
with tf.variable_scope('loss'):
self.seq2seq_loss = l1_loss(seq2seq_gtruth, seq2seq_output)
self.post_loss = l1_loss(post_gtruth, post_output)
self.loss = self.seq2seq_loss + self.post_loss
def __step_while_loop(self, step_count: int,
truth_outputs_padded: tf.Tensor,
truth_outputs_counts: tf.Tensor,
initial_hidden_vector: tf.Tensor):
def create_guess_layers(parent_hidden_vector: tf.Tensor,
child_hidden_vector: tf.Tensor,
inner_hidden_vector: tf.Tensor,
state_number: int):
state = self.object_type.get_all_states()[state_number]
return self.hidden_vector_network(parent_hidden_vector,
child_hidden_vector,
inner_hidden_vector, state)
def body(step: int, stack_1, stack_2, states_ta: tf.TensorArray,
outputs_ta: tf.TensorArray, outputs_counts_ta: tf.TensorArray,
return_value: tf.Tensor):
# stack_1 = tf.Print(stack_1, [step, tf.slice(stack_1, [0, 1], [-1, 2])], "stack: ", summarize=100)
# Rebuild `stack` tuple
# TODO: Find way to avoid this by putting tuples into `tf.while_loop` arguments
stack = stack_1, stack_2
# Get the state and hidden vector we have to deal with for this iteration
state, hidden_vector, stack = state_stack.pop(stack)
# Get the summary for all hidden vectors excluding this one
hidden_vector_summary = state_stack.get_hidden_vector_summary(
stack)
# Get the number of outputs for padding
# TODO: Doing this twice, once here, and once when calling create_guess_layers. Fix this
num_outputs = tf.case(pred_fn_pairs=[
(tf.equal(state, i), lambda i=i: tf.constant(
self.object_type.get_all_states()[i].num_outputs))
for i in range(len(self.object_type.get_all_states()))
],
default=lambda: tf.constant(0))
# Call `create_guess_layers(...)` depending on what state we're in
next_hidden_vector, current_choice = tf.case(
pred_fn_pairs=[
(tf.equal(state, i), lambda i=i: create_guess_layers(
hidden_vector_summary, return_value, hidden_vector, i))
for i in range(len(self.object_type.get_all_states()))
],
default=lambda: (hidden_vector, tf.constant(
0, dtype=tf.float32) / tf.constant(0, tf.float32)))
# Zero pad the current choice
current_choice = tf.concat([
current_choice,
tf.zeros([self.max_outputs - tf.shape(current_choice)[0]])
],
axis=0,
name="current_choice_zero_padded")
# Reshape the hidden vector so we know what size it is
next_hidden_vector = tf.reshape(next_hidden_vector,
[self.hidden_vector_size],
name="next_hidden_vector_reshaped")
if self.training:
# If we're training, the choice we send to the update_state_stack_fn should be determined by the truth
stack_update_choice = tf.gather(truth_outputs_padded,
step,
name="choice_from_input")
else:
# Otherwise, the choice should be what we outputted
stack_update_choice = current_choice
# stack = (tf.Print(stack[0], [step, stack[0][:stack_2]], "estack: ", summarize=100), stack[1])
# Update the state stack
stack, return_value = self.__update_state_stack(
state, stack, next_hidden_vector, stack_update_choice)
return \
step + 1, \
(*stack), \
states_ta.write(step, state, "write_state"), \
outputs_ta.write(step, current_choice, "write_outputs"), \
outputs_counts_ta.write(step, num_outputs, "write_outputs_count"), \
return_value
def cond(step, stack_1, stack_2, *_):
# Rebuild `stack` tuple
stack = stack_1, stack_2
if self.training:
return step < step_count
else:
return tf.logical_and(
step < self.max_steps,
tf.not_equal(state_stack.check_size(stack, 1), True))
# Create the initial stack with initial hidden vector and state
initial_stack = state_stack.create(
max_size=self.max_steps,
hidden_vector_size=self.hidden_vector_size)
initial_stack = state_stack.push(initial_stack, -1,
initial_hidden_vector)
initial_stack = state_stack.push(
initial_stack,
self.object_type.get_initial_state().id, initial_hidden_vector)
# If we're training we know the output sizes and don't need larger than necessary `TensorArray`s
output_sizes = step_count if self.training else self.max_steps
# Create the `tf.TensorArray` to hold all outputs
initial_states_ta = tf.TensorArray(dtype=tf.float32,
size=output_sizes,
name="initial_states_ta")
initial_outputs_ta = tf.TensorArray(dtype=tf.float32,
size=output_sizes,
name="initial_outputs_ta")
initial_outputs_counts_ta = tf.TensorArray(
dtype=tf.int32,
size=output_sizes,
name="initial_outputs_counts_ta")
initial_return_value = tf.constant(np.nan,
dtype=tf.float32,
shape=[self.hidden_vector_size])
final_step, *_, final_states_ta, final_outputs_ta, final_outputs_counts_ta, _ = tf.while_loop(
cond=cond,
body=body,
loop_vars=[
0, *initial_stack, initial_states_ta, initial_outputs_ta,
initial_outputs_counts_ta, initial_return_value
],
shape_invariants=[
tf.TensorShape([]),
tf.TensorShape([None, self.hidden_vector_size + 1]),
tf.TensorShape([]),
tf.TensorShape(None),
tf.TensorShape(None),
tf.TensorShape(None),
tf.TensorShape([self.hidden_vector_size])
],
name="step_while_loop")
# If in test mode, we don't know the amount of steps we will execute. So we need to resize the `tf.TensorArray`
# to match the actual output
# TODO: Check if we can overcome this by using padding
if not self.training:
final_states_ta = tf_utils.resize_tensor_array(
final_states_ta, final_step)
final_outputs_ta = tf_utils.resize_tensor_array(
final_outputs_ta, final_step)
final_outputs_counts_ta = tf_utils.resize_tensor_array(
final_outputs_counts_ta, final_step)
return \
self.__stack_and_pad(final_states_ta, self.max_steps), \
self.__stack_and_pad(final_outputs_ta, self.max_steps), \
self.__stack_and_pad(final_outputs_counts_ta, self.max_steps), \
final_step
