def std_forward(a, weights, bias_weights, name=None):
with tf.op_scope([a, W, Wb], name, 'std_forward') as scope:
a = tf.convert_to_tensor(a, dtype=tf.float32, name='input')
weights = tf.convert_to_tensor(weights, dtype=tf.float32, name='weights')
bias_weights = tf.convert_to_tensor(bias_weights, dtype=tf.float32, name='bias_weights')
biased = tf.concat(1, (weights, bias_weights), name='biased')
return tf.matmul(biased, a, name=scope)
def test_regression_metrics():
y_a = tf.convert_to_tensor(np.random.random((6, 7)))
y_b = tf.convert_to_tensor(np.random.random((6, 7)))
for metric in all_regression_metrics:
output = metric(y_a, y_b)
with sess.as_default():
assert output.eval().shape == ()
def testMismatchedDimensions(self):
for func in [tf.add, tf.sub, tf.mul, tf.div, _ADD, _SUB, _MUL, _TRUEDIV,
_FLOORDIV]:
with self.assertRaisesWithPredicateMatch(
ValueError, lambda e: "Incompatible shapes" in e.message):
func(tf.convert_to_tensor([10.0, 20.0, 30.0]),
tf.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
def forward_prop_nodes(i_start, size, acts, offset):
# Note: In the corpus that we've seen, parse trees are always ordered such that
# iteration forward through the list will be in bottom-up order.
# Conversely, iteration in reverse is always top-down.
# This enables a simple iterative algorithm. If this were not the case,
# putting the nodes in order by a postorder traversal would fix it.
def fwd_continue(*parms):
(_, sz, cur, _) = parms
return tf.less(cur, sz, name='cur_le_size')
def forward_prop(*parms):
(i0, sz, cur, act) = parms
with tf.device('/gpu:0'):
gact = act
gcur = cur
next_idx = i0 + gcur
node_out = tf.reshape(forward_node(next_idx, act, offset), [1, FLAGS.wvs, 1], name='node_out')
tf.scatter_add(gact, tf.pack([gcur]), node_out, name='act_update')
act = gact
return [i0, sz, cur + iONE, act]
with tf.device('/cpu:0'):
i_start = tf.convert_to_tensor(i_start, dtype=tf.int32, name='i_start')
size = tf.convert_to_tensor(size, dtype=tf.int32, name='size')
iZ = tf.convert_to_tensor(0, dtype=tf.int32, name='ZERO')
while_parms = [i_start, size, iZ, acts]
wresult = tf.while_loop(fwd_continue, forward_prop, while_parms, parallel_iterations=1,
name='forward_prop_while')
(_, _, _, result) = wresult
return tf.slice(result, [0, 0, 0], tf.pack([size, -1, -1]), name='fwd_prop_nodes')
def one_minus_pseudo_unitcell_transfer_op(direction, mps, left_dominant,
right_dominant, vector):
"""
calculates action of 11-Transfer-Operator +|r)(l|
Parameters:
---------------------------
direction: int or str
if (1,'l','left'): do left multiplication
if (-1,'r','right'): do right multiplication
mps: InfiniteMPSCentralGauge object
an infinite mps
left_dominant: tf.tensor of shape (mps.D[0],mps.D[0])
left dominant eigenvector of the unit-cell transfer operator of mps
right_dominant: tf.tensor of shape (mps.D[-1],mps.D[-1])
right dominant eigenvector of the unit-cell transfer operator of mps
vector: tf.tensor of shape (mps.D[0]*mps.D[0]) or (mps.D[-1]*mps.D[-1])
the input vector
Returns
---------------------------
np.ndarray of shape (mps.D[0]*mps.D[0]) or (mps.D[-1]*mps.D[-1])
"""
if direction in (1, 'l', 'left'):
x = tf.reshape(tf.convert_to_tensor(vector), (mps.D[0], mps.D[0]))
temp = x - mps.unitcell_transfer_op('left', x) + ncon(
[x, right_dominant], [[1, 2], [1, 2]]) * left_dominant
return tf.reshape(temp, [mps.D[-1] * mps.D[-1]]).numpy()
if direction in (-1, 'r', 'right'):
x = tf.reshape(tf.convert_to_tensor(vector), [mps.D[-1], mps.D[-1]])
temp = x - mps.unitcell_transfer_op('right', x) + ncon(
[left_dominant, x], [[1, 2], [1, 2]]) * right_dominant
return tf.reshape(temp, [mps.D[0] * mps.D[0]]).numpy()
def testGradient(self):
s = [2, 3, 4, 2]
# NOTE(kearnes): divide by 20 so product is a reasonable size
x = np.arange(1.0, 49.0).reshape(s).astype(np.float32) / 20.
with self.test_session():
t = tf.convert_to_tensor(x)
su = tf.reduce_prod(t, [])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [2, 3, 4, 2], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [1, 2])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [2, 2], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [0, 1, 2, 3])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [1], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
# NOTE(kearnes): the current gradient calculation gives NaNs for 0 inputs
x = np.arange(0.0, 48.0).reshape(s).astype(np.float32) / 20.
with self.test_session():
t = tf.convert_to_tensor(x)
su = tf.reduce_prod(t, [])
jacob_t, _ = gradient_checker.ComputeGradient(
t, s, su, [2, 3, 4, 2], x_init_value=x, delta=1)
with self.assertRaisesOpError("Tensor had NaN values"):
tf.check_numerics(jacob_t, message="_ProdGrad NaN test").op.run()
def testLSTMBasicToBlockPeeping(self):
with self.test_session(use_gpu=self._use_gpu) as sess:
batch_size = 2
input_size = 3
cell_size = 4
sequence_length = 5
inputs = []
for _ in range(sequence_length):
inp = tf.convert_to_tensor(
np.random.randn(batch_size, input_size),
dtype=tf.float32)
inputs.append(inp)
initializer = tf.random_uniform_initializer(-0.01, 0.01, seed=19890212)
with tf.variable_scope("basic", initializer=initializer):
cell = tf.nn.rnn_cell.LSTMCell(cell_size,
use_peepholes=True,
state_is_tuple=True)
outputs, _ = tf.nn.rnn(cell, inputs, dtype=tf.float32)
sess.run([tf.initialize_all_variables()])
basic_outputs = sess.run(outputs)
basic_grads = sess.run(tf.gradients(outputs, inputs))
basic_wgrads = sess.run(tf.gradients(outputs, tf.trainable_variables()))
with tf.variable_scope("block", initializer=initializer):
w = tf.get_variable("w",
shape=[input_size + cell_size, cell_size * 4],
dtype=tf.float32)
b = tf.get_variable("b",
shape=[cell_size * 4],
dtype=tf.float32,
initializer=tf.zeros_initializer)
wci = tf.get_variable("wci", shape=[cell_size], dtype=tf.float32)
wcf = tf.get_variable("wcf", shape=[cell_size], dtype=tf.float32)
wco = tf.get_variable("wco", shape=[cell_size], dtype=tf.float32)
_, _, _, _, _, _, outputs = fused_lstm(
tf.convert_to_tensor(sequence_length,
dtype=tf.int64),
inputs,
w,
b,
wci=wci,
wcf=wcf,
wco=wco,
cell_clip=0,
use_peephole=True)
sess.run([tf.initialize_all_variables()])
block_outputs = sess.run(outputs)
block_grads = sess.run(tf.gradients(outputs, inputs))
block_wgrads = sess.run(tf.gradients(outputs, [w, b, wci, wcf, wco]))
self.assertAllClose(basic_outputs, block_outputs)
self.assertAllClose(basic_grads, block_grads)
for basic, block in zip(basic_wgrads, block_wgrads):
self.assertAllClose(basic, block, rtol=1e-2, atol=1e-2)
def embedding_lookup(params, ids, name="embedding_lookup"):
"""Provides a N dimensional version of tf.embedding_lookup.
Ids are flattened to a 1d tensor before being passed to embedding_lookup
then, they are unflattend to match the original ids shape plus an extra
leading dimension of the size of the embeddings.
Args:
params: List of tensors of size D0 x D1 x ... x Dn-2 x Dn-1.
ids: N-dimensional tensor of B0 x B1 x .. x Bn-2 x Bn-1.
Must contain indexes into params.
name: Optional name for the op.
Returns:
A tensor of size B0 x B1 x .. x Bn-2 x Bn-1 x D1 x ... x Dn-2 x Dn-1 containing the values from
the params tensor(s) for indecies in ids.
Raises:
ValueError: if some parameters are invalid.
"""
with tf.op_scope([params, ids], name, "embedding_lookup"):
params = tf.convert_to_tensor(params)
ids = tf.convert_to_tensor(ids)
shape = tf.shape(ids)
ids_flat = tf.reshape(ids, tf.reduce_prod(shape, keep_dims=True))
embeds_flat = tf.nn.embedding_lookup(params, ids_flat, name)
embed_shape = tf.concat(0, [shape, [-1]])
embeds = tf.reshape(embeds_flat, embed_shape)
embeds.set_shape(ids.get_shape().concatenate(params.get_shape()[1:]))
return embeds
def _prepare_args_with_initial_simplex(objective_function,
initial_simplex,
objective_at_initial_simplex,
batch_evaluate_objective):
"""Evaluates the objective function at the specified initial simplex."""
initial_simplex = tf.convert_to_tensor(initial_simplex)
# If d is the dimension of the problem, the number of vertices in the
# simplex should be d+1. From this, we can infer the number of dimensions
# as n - 1 where n is the number of vertices specified.
num_vertices = tf.shape(initial_simplex)[0]
dim = num_vertices - 1
num_evaluations = 0
if objective_at_initial_simplex is None:
objective_at_initial_simplex, n_evals = _evaluate_objective_multiple(
objective_function, initial_simplex, batch_evaluate_objective)
num_evaluations += n_evals
objective_at_initial_simplex = tf.convert_to_tensor(
objective_at_initial_simplex)
return (dim,
num_vertices,
initial_simplex,
objective_at_initial_simplex,
num_evaluations)
def make_multivariate_mixture(batch_shape, num_components, event_shape,
use_static_graph, batch_shape_tensor=None):
if batch_shape_tensor is None:
batch_shape_tensor = batch_shape
batch_shape_tensor = tf.convert_to_tensor(batch_shape_tensor, tf.int32)
logits = tf.random_uniform(
tf.concat((batch_shape_tensor, [num_components]), 0),
-1,
1,
dtype=tf.float32) - 50.
logits.set_shape(tf.TensorShape(batch_shape).concatenate(num_components))
static_batch_and_event_shape = (
tf.TensorShape(batch_shape).concatenate(event_shape))
event_shape = tf.convert_to_tensor(event_shape, tf.int32)
batch_and_event_shape = tf.concat((batch_shape_tensor, event_shape), 0)
def create_component():
loc = tf.random_normal(batch_and_event_shape)
scale_diag = 10 * tf.random_uniform(batch_and_event_shape)
loc.set_shape(static_batch_and_event_shape)
scale_diag.set_shape(static_batch_and_event_shape)
return tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale_diag)
components = [create_component() for _ in range(num_components)]
cat = tfd.Categorical(logits, dtype=tf.int32)
return tfd.Mixture(cat, components, use_static_graph=use_static_graph)
def testOneOpCond(self):
with self.test_session():
v = tf.Variable(0)
c = tf.convert_to_tensor(0)
one = tf.convert_to_tensor(1)
two = tf.convert_to_tensor(2)
p = tf.greater_equal(c, 1)
def a():
return tf.assign(v, one)
def b():
return tf.assign(v, two)
i = tf.cond(p, a, b)
self.assertTrue(isinstance(i, tf.Tensor))
tf.initialize_all_variables().run()
self.assertEqual(0, v.eval())
# True case: c = 2 is >= 1, v is set to 1.
self.assertEqual(1, i.eval(feed_dict={c.name: 2}))
self.assertEqual(1, v.eval())
# False case: c = 0 is not >= 1, v is set to 2.
self.assertEqual(2, i.eval(feed_dict={c.name: 0}))
self.assertEqual(2, v.eval())
def _teacher_forcing_ratio_decay(init_tfr, global_step, hparams): ################################################################# # Narrow Cosine Decay: # Phase 1: tfr = 1 # We only start learning rate decay after 10k steps # Phase 2: tfr in ]0, 1[ # decay reach minimal value at step ~280k # Phase 3: tfr = 0 # clip by minimal teacher forcing ratio value (step >~ 280k) ################################################################# #Compute natural cosine decay tfr = tf.train.cosine_decay(init_tfr, global_step=global_step - hparams.tacotron_teacher_forcing_start_decay, #tfr = 1 at step 10k decay_steps=hparams.tacotron_teacher_forcing_decay_steps, #tfr = 0 at step ~280k alpha=hparams.tacotron_teacher_forcing_decay_alpha, #tfr = 0% of init_tfr as final value name='tfr_cosine_decay') #force teacher forcing ratio to take initial value when global step < start decay step. narrow_tfr = tf.cond( tf.less(global_step, tf.convert_to_tensor(hparams.tacotron_teacher_forcing_start_decay)), lambda: tf.convert_to_tensor(init_tfr), lambda: tfr) return narrow_tfr
def testSumGradArgs(self):
with self.test_session(use_gpu=False):
indices = [tf.convert_to_tensor([0, 1, 2, 3]),
tf.convert_to_tensor([2, 3])]
values = [tf.convert_to_tensor([2, 3, 5, 7]), tf.convert_to_tensor([1, 1])]
self.assertAllEqual(
tf.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1])
def test_multivariate_normal(session_tf, x, mu, cov_sqrt):
cov = np.dot(cov_sqrt, cov_sqrt.T)
L = np.linalg.cholesky(cov)
if len(x.shape) != 2 or len(mu.shape) != 2:
with pytest.raises(Exception) as e_info:
gp_result = logdensities.multivariate_normal(
tf.convert_to_tensor(x),
tf.convert_to_tensor(mu),
tf.convert_to_tensor(L))
else:
x_tf = tf.placeholder(settings.float_type)
mu_tf = tf.placeholder(settings.float_type)
gp_result = logdensities.multivariate_normal(
x_tf, mu_tf, tf.convert_to_tensor(L))
gp_result = session_tf.run(gp_result, feed_dict={x_tf: x, mu_tf: mu})
if mu.shape[1] > 1:
if x.shape[1] > 1:
sp_result = [mvn.logpdf(x[:,i], mu[:,i], cov) for i in range(mu.shape[1])]
else:
sp_result = [mvn.logpdf(x.ravel(), mu[:, i], cov) for i in range(mu.shape[1])]
else:
sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
assert_allclose(gp_result, sp_result)
def test_CE_loss(sess, CE_arrays):
y, y_hat = CE_arrays
y = tf.convert_to_tensor(y, dtype=tf.float64)
y_hat = tf.convert_to_tensor(y_hat, dtype=tf.float64)
sess.run(cross_entropy_loss(y,y_hat))
assert 1
print("CE_loss ran to completion")
def test_encode_annos(self):
with self.test_session() as sess:
num_obj = 2
image_shape = [config.IMG_HEIGHT, config.IMG_WIDTH]
fea_shape = [config.FEA_HEIGHT, config.FEA_WIDTH]
num_classes = config.NUM_CLASSES
images = tf.constant(0, shape=[image_shape[0], image_shape[1], 3])
labels = tf.constant(1, shape=[num_obj])
anchors = set_anchors(image_shape, fea_shape)
# Construct test bbox
bbox_1 = tf.convert_to_tensor(xywh_to_yxyx(anchors[0][0][0]), dtype=tf.float32)
bbox_2 = tf.convert_to_tensor(xywh_to_yxyx(anchors[2][2][1]), dtype=tf.float32)
bboxes = tf.stack([bbox_1, bbox_2], axis=0)
input_mask, labels_input, box_delta_input, box_input = \
encode_annos(images, labels, bboxes, anchors, num_classes)
out_input_mask, out_labels_input, out_box_delta_input, out_box_input = \
sess.run([input_mask, labels_input, box_delta_input, box_input])
print("input_mask:", out_input_mask)
print("box_input:", out_box_input)
print("label_input:", out_labels_input)
print("box_delta_input:", out_box_delta_input)
print("shape:",
"input_mask:", np.shape(out_input_mask),
"labels_input:", np.shape(out_labels_input),
"box_delta_input:", np.shape(out_box_delta_input),
"box_input:", np.shape(out_box_input)
)
def batched_index(values, indices):
"""Equivalent to `values[:, indices]`.
Performs indexing on batches and sequence-batches by reducing over
zero-masked values. Compared to indexing with `tf.gather` this approach is
more general and TPU-friendly, but may be less efficient if `num_values`
is large. It works with tensors whose shapes are unspecified or
partially-specified, but this op will only do shape checking on shape
information available at graph construction time. When complete shape
information is absent, certain shape incompatibilities may not be detected at
runtime! See `indexing_ops_test` for detailed examples.
Args:
values: tensor of shape `[B, num_values]` or `[T, B, num_values]`
indices: tensor of shape `[B]` or `[T, B]` containing indices.
Returns:
Tensor of shape `[B]` or `[T, B]` containing values for the given indices.
Raises: ValueError if values and indices have sizes that are known
statically (i.e. during graph construction), and those sizes are not
compatible (see shape descriptions in Args list above).
"""
with tf.name_scope("batch_indexing", values=[values, indices]):
values = tf.convert_to_tensor(values)
indices = tf.convert_to_tensor(indices)
assert_compatible_shapes(values.shape, indices.shape)
one_hot_indices = tf.one_hot(
indices, tf.shape(values)[-1], dtype=values.dtype)
return tf.reduce_sum(values * one_hot_indices, axis=-1)
def get_hit_rate_and_ndcg(self, predicted_scores_by_user, items_by_user,
top_k=rconst.TOP_K, match_mlperf=False):
rconst.TOP_K = top_k
rconst.NUM_EVAL_NEGATIVES = predicted_scores_by_user.shape[1] - 1
batch_size = items_by_user.shape[0]
users = np.repeat(np.arange(batch_size)[:, np.newaxis],
rconst.NUM_EVAL_NEGATIVES + 1, axis=1)
users, items, duplicate_mask = \
data_pipeline.BaseDataConstructor._assemble_eval_batch(
users, items_by_user[:, -1:], items_by_user[:, :-1], batch_size)
g = tf.Graph()
with g.as_default():
logits = tf.convert_to_tensor(
predicted_scores_by_user.reshape((-1, 1)), tf.float32)
softmax_logits = tf.concat([tf.zeros(logits.shape, dtype=logits.dtype),
logits], axis=1)
duplicate_mask = tf.convert_to_tensor(duplicate_mask, tf.float32)
metric_ops = neumf_model.compute_eval_loss_and_metrics(
logits=logits, softmax_logits=softmax_logits,
duplicate_mask=duplicate_mask, num_training_neg=NUM_TRAIN_NEG,
match_mlperf=match_mlperf).eval_metric_ops
hr = metric_ops[rconst.HR_KEY]
ndcg = metric_ops[rconst.NDCG_KEY]
init = [tf.global_variables_initializer(),
tf.local_variables_initializer()]
with self.test_session(graph=g) as sess:
sess.run(init)
return sess.run([hr[1], ndcg[1]])
def _get_sampling_probability(hparams, is_training):
"""Returns `sampling_probabiliy` if `sampling schedule` given or 0."""
if (not hasattr(hparams, 'sampling_schedule') or
not hparams.sampling_schedule):
return tf.convert_to_tensor(0.0, tf.float32)
if not is_training:
# This is likely an eval/test job associated with a training job using
# scheduled sampling.
tf.logging.warning(
'Setting non-training sampling schedule from %s:%f to constant:1.0.',
hparams.sampling_schedule, hparams.sampling_rate)
hparams.sampling_schedule = 'constant'
hparams.sampling_rate = 1.0
if hparams.sampling_schedule == 'constant':
sampling_probability = tf.constant(hparams.sampling_rate)
elif hparams.sampling_schedule == 'inverse_sigmoid':
k = tf.constant(hparams.sampling_rate)
sampling_probability = 1.0 - (
k / (k + tf.exp(tf.to_float(tf.train.get_or_create_global_step()) / k)))
elif hparams.sampling_schedule == 'exponential':
if not 0 < hparams.sampling_rate < 1:
raise ValueError(
'Exponential sampling rate must be in the interval (0, 1). Got %f.'
% hparams.sampling_rate)
k = tf.constant(hparams.sampling_rate)
sampling_probability = (
1.0 - tf.pow(k, tf.to_float(tf.train.get_or_create_global_step())))
else:
tf.logging.fatal('Invalid sampling_schedule: %s',
hparams.sampling_schedule)
tf.summary.scalar('sampling_probability', sampling_probability)
return tf.convert_to_tensor(sampling_probability, tf.float32)
def testSingleExampleWithSparseAndDense(self):
original = example(features=features(
{"c": float_feature([3, 4]),
"st_a": float_feature([3.0, 4.0])}))
serialized = original.SerializeToString()
expected_st_a = (
np.array([[0], [1]], dtype=np.int64), # indices
np.array([3.0, 4.0], dtype=np.float32), # values
np.array([2], dtype=np.int64)) # shape: max_values = 2
a_default = [1, 2, 3]
b_default = np.random.rand(3, 3).astype(bytes)
expected_output = {
"st_a": expected_st_a,
"a": [a_default],
"b": b_default,
"c": np.array([3, 4], dtype=np.float32),
}
self._test({
"example_names": tf.convert_to_tensor("in1"),
"serialized": tf.convert_to_tensor(serialized),
"features": {
"st_a": tf.VarLenFeature(tf.float32),
"a": tf.FixedLenFeature((1, 3), tf.int64, default_value=a_default),
"b": tf.FixedLenFeature((3, 3), tf.string, default_value=b_default),
# Feature "c" must be provided, since it has no default_value.
"c": tf.FixedLenFeature((2,), tf.float32),
}
}, expected_output)
def histogram(self, x, value_range=None, nbins=None, name=None):
"""Return histogram of values.
Given the tensor `values`, this operation returns a rank 1 histogram
counting the number of entries in `values` that fell into every bin. The
bins are equal width and determined by the arguments `value_range` and
`nbins`.
Args:
x: 1D numeric `Tensor` of items to count.
value_range: Shape [2] `Tensor`. `new_values <= value_range[0]` will be
mapped to `hist[0]`, `values >= value_range[1]` will be mapped to
`hist[-1]`. Must be same dtype as `x`.
nbins: Scalar `int32 Tensor`. Number of histogram bins.
name: Python `str` name prefixed to Ops created by this class.
Returns:
counts: 1D `Tensor` of counts, i.e.,
`counts[i] = sum{ edges[i-1] <= values[j] < edges[i] : j }`.
edges: 1D `Tensor` characterizing intervals used for counting.
"""
with tf.name_scope(name, "histogram", [x]):
x = tf.convert_to_tensor(x, name="x")
if value_range is None:
value_range = [tf.reduce_min(x), 1 + tf.reduce_max(x)]
value_range = tf.convert_to_tensor(value_range, name="value_range")
lo = value_range[0]
hi = value_range[1]
if nbins is None:
nbins = tf.to_int32(hi - lo)
delta = (hi - lo) / tf.cast(nbins, dtype=value_range.dtype.base_dtype)
edges = tf.range(
start=lo, limit=hi, delta=delta, dtype=x.dtype.base_dtype)
counts = tf.histogram_fixed_width(x, value_range=value_range, nbins=nbins)
return counts, edges
def __init__(self,
loc=0.,
scale=1.,
validate_args=False,
name="gumbel"):
"""Instantiates the `Gumbel` bijector.
Args:
loc: Float-like `Tensor` that is the same dtype and is
broadcastable with `scale`.
This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
scale: Positive Float-like `Tensor` that is the same dtype and is
broadcastable with `loc`.
This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
validate_args: Python `bool` indicating whether arguments should be
checked for correctness.
name: Python `str` name given to ops managed by this object.
"""
self._graph_parents = []
self._name = name
self._validate_args = validate_args
with self._name_scope("init", values=[loc, scale]):
self._loc = tf.convert_to_tensor(loc, name="loc")
self._scale = tf.convert_to_tensor(scale, name="scale")
tf.assert_same_float_dtype([self._loc, self._scale])
if validate_args:
self._scale = control_flow_ops.with_dependencies([
tf.assert_positive(
self._scale, message="Argument scale was not positive")
], self._scale)
super(Gumbel, self).__init__(
validate_args=validate_args,
forward_min_event_ndims=0,
name=name)
def file_to_dataset(filepath, N, vectorizer):
#convert file to dataset, returns X and Y tensors of integer indexes describing the N words (X) leading up to (Y)
f = open(filepath)
lines = f.readlines()
X = []
Y = []
line_index = 0
for line in lines:
if line_index % 1000 == 0:
print(str(line_index) +'/' len(lines) + ' lines read...')
line_index +=1
words = line.split()
padding = (N-1)*['<pre>']
#remove /n at the end, add padding to front
words = padding + words[:-1]
for i in range(0, len(words) - N):
x = []
y = vectorizer.transform(words[i+N]).toarray()
x_words = words[i:i+N]
for word in x_words:
x.append(vectorizer.transform(word).toarray())
X.append(x)
Y.append(y)
return tf.convert_to_tensor(X), tf.convert_to_tensor(Y)
def log_prob(self, xs, zs):
"""
Parameters
----------
xs : dict of str to tf.Tensor
Data dictionary. Each key is a data structure used in the
model (Theano shared variable), and its value is the
corresponding realization (tf.Tensor).
zs : dict of str to tf.Tensor
Latent variable dictionary. Each key names a latent variable
used in the model (str), and its value is the corresponding
realization (tf.Tensor).
Returns
-------
tf.Tensor
Scalar, the log joint density log p(xs, zs).
Notes
-----
It wraps around a Python function. The Python function takes
inputs of type np.ndarray and outputs a np.ndarray.
"""
# Store keys so that ``_py_log_prob_args`` knows how each
# value corresponds to a key.
self.xs_keys = list(six.iterkeys(xs))
self.zs_keys = list(six.iterkeys(zs))
# Pass in all tensors as a flattened list for tf.py_func().
inputs = [tf.convert_to_tensor(x) for x in six.itervalues(xs)]
inputs += [tf.convert_to_tensor(z) for z in six.itervalues(zs)]
return tf.py_func(self._py_log_prob_args, inputs, [tf.float32])[0]
def __init__(self, data_dir, data_list, input_size,
random_scale, random_mirror, ignore_label, img_mean, coord):
'''Initialise an ImageReader.
Args:
data_dir: path to the directory with images and masks.
data_list: path to the file with lines of the form '/path/to/image /path/to/mask'.
input_size: a tuple with (height, width) values, to which all the images will be resized.
random_scale: whether to randomly scale the images prior to random crop.
random_mirror: whether to randomly mirror the images prior to random crop.
ignore_label: index of label to ignore during the training.
img_mean: vector of mean colour values.
coord: TensorFlow queue coordinator.
'''
self.data_dir = data_dir
self.data_list = data_list
self.input_size = input_size
self.coord = coord
self.image_list, self.label_list = read_labeled_image_list(self.data_dir, self.data_list)
self.images = tf.convert_to_tensor(self.image_list, dtype=tf.string)
self.labels = tf.convert_to_tensor(self.label_list, dtype=tf.string)
self.queue = tf.train.slice_input_producer([self.images, self.labels],
shuffle=input_size is not None) # not shuffling if it is val
self.image, self.label = read_images_from_disk(self.queue, self.input_size, random_scale, random_mirror, ignore_label, img_mean)
def test_regression_metrics(self):
with self.test_session():
y_a = tf.convert_to_tensor(np.random.random((6, 7)))
y_b = tf.convert_to_tensor(np.random.random((6, 7)))
for metric in all_regression_metrics:
output = metric(y_a, y_b)
assert output.eval().shape == ()
def testPlaceholder(self):
with self.test_session(use_gpu=True):
# Test using placeholder with a defined shape.
ph_0 = tf.placeholder(tf.int32, shape=[])
result_0 = tf.convert_to_tensor([[0, 0, 0],
[0, ph_0, 0],
[0, 0, 0]])
self.assertAllEqual([[0, 0, 0],
[0, 1, 0],
[0, 0, 0]],
result_0.eval(feed_dict={ph_0: 1}))
self.assertAllEqual([[0, 0, 0],
[0, 2, 0],
[0, 0, 0]],
result_0.eval(feed_dict={ph_0: 2}))
# Test using placeholder with an undefined shape.
ph_1 = tf.placeholder(tf.int32)
result_1 = tf.convert_to_tensor([[0, 0, 0],
[0, ph_1, 0],
[0, 0, 0]])
self.assertAllEqual([[0, 0, 0], [0, 1, 0], [0, 0, 0]],
result_1.eval(feed_dict={ph_1: 1}))
self.assertAllEqual([[0, 0, 0], [0, 2, 0], [0, 0, 0]],
result_1.eval(feed_dict={ph_1: 2}))
def _compare(self, func, x, y, dtype):
with self.test_session(use_gpu=False):
out = func(
tf.convert_to_tensor(np.array([x]).astype(dtype)), tf.convert_to_tensor(np.array([y]).astype(dtype))
)
ret = out.eval()
return ret[0]
def testLoop_1(self):
with self.test_session():
zero = tf.convert_to_tensor(0)
one = tf.convert_to_tensor(1)
n = tf.constant(10)
enter_zero = control_flow_ops.enter(zero, "foo_1", False)
enter_one = control_flow_ops.enter(one, "foo_1", False)
enter_n = control_flow_ops.enter(n, "foo_1", False)
merge_zero = control_flow_ops.merge([enter_zero, enter_zero], name="merge_zero")[0]
merge_one = control_flow_ops.merge([enter_one, enter_one], name="merge_one")[0]
merge_n = control_flow_ops.merge([enter_n, enter_n], name="merge_n")[0]
less_op = tf.less(merge_n, merge_n)
cond_op = control_flow_ops.loop_cond(less_op)
switch_zero = control_flow_ops.switch(merge_zero, cond_op)
switch_one = control_flow_ops.switch(merge_one, cond_op)
switch_n = control_flow_ops.switch(merge_n, cond_op)
next_zero = control_flow_ops.next_iteration(switch_zero[1])
next_one = control_flow_ops.next_iteration(switch_one[1])
next_n = control_flow_ops.next_iteration(switch_n[1])
merge_zero.op._update_input(1, next_zero)
merge_one.op._update_input(1, next_one)
merge_n.op._update_input(1, next_n)
exit_n = control_flow_ops.exit(switch_n[0])
result = exit_n.eval()
self.assertAllEqual(10, result)
def check_split_latent_conditioning(self, merge_std):
with tf.Graph().as_default():
rng = np.random.RandomState(0)
x_rand = rng.randn(12, 32, 32, 32).astype(np.float32)
latent_rand = rng.randn(12, 32, 32, 16).astype(np.float32)
x_t = tf.convert_to_tensor(x_rand)
latent_t = tf.convert_to_tensor(latent_rand)
hparams = glow.glow_hparams()
hparams.level_scale = merge_std
hparams.add_hparam("latent_dist_encoder", "pointwise")
# Test initalization.
# x2 ~ N(scale * latent, 1.0) where initial scale is 1.0
exp_x2 = x_rand[:, :, :, 16:]
exp_eps = x_rand[:, :, :, 16:] - latent_rand
x_inv, _, eps, x2_t, _ = glow_ops.split(
merge_std, x_t, cond_latents=latent_t, hparams=hparams,
condition=True)
# Test reversibility.
x_inv_inv, _, _ = glow_ops.split(
merge_std, x_inv, cond_latents=latent_t, eps=eps, reverse=True,
hparams=hparams, condition=True)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
actual_eps, actual_x2, diff_np = sess.run([eps, x2_t, x_inv_inv - x_t])
self.assertTrue(np.allclose(diff_np, 0.0, atol=1e-5))
self.assertTrue(np.allclose(actual_eps, exp_eps))
self.assertTrue(np.allclose(exp_x2, actual_x2))
def setup(self):
"""
This method set the appropriate variables necessary for the random 2D augmentation. It also computes the
transformation matrix.
Returns:
None
"""
# \NOTE(JP): tracing behavior from dataset.map causes issue when any tensor id defined as tf.constant
transform_matrix = tf.convert_to_tensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=tf.float32
)
do_rotate = False
do_shift = False
do_zoom = False
do_shear = False
self.do_flip_lr_tensor = tf.convert_to_tensor(False)
self.do_flip_ud_tensor = tf.convert_to_tensor(False)
if type(self.rotation_range) is not tuple and type(self.rotation_range) is not list:
if self.rotation_range > 0.:
do_rotate = True
else:
if self.rotation_range[0] > 0. or self.rotation_range[1] > 0.:
do_rotate = True
if type(self.width_shift_range) is not tuple and type(self.width_shift_range) is not list:
if self.width_shift_range > 0.:
do_shift = True
else:
if self.width_shift_range[0] > 0. or self.width_shift_range[1] > 0.:
do_shift = True
if type(self.height_shift_range) is not tuple and type(self.height_shift_range) is not list:
if self.height_shift_range > 0.:
do_shift = True
else:
if self.height_shift_range[0] > 0. or self.height_shift_range[1] > 0.:
do_shift = True
if type(self.zoom_range) is not tuple and type(self.zoom_range) is not list:
if self.zoom_range != 1.:
do_zoom = True
else:
if self.zoom_range[0] != 1. or self.zoom_range[1] != 0.:
do_zoom = True
if type(self.shear_range) is not tuple and type(self.shear_range) is not list:
if self.shear_range > 0.:
do_shear = True
else:
if self.shear_range[0] > 0. or self.shear_range[1] > 0.:
do_shear = True
if do_rotate:
if transform_matrix is None:
transform_matrix = self.rotate()
else:
transform_matrix = tf.tensordot(transform_matrix, self.rotate(), axes=1)
if do_shift:
if transform_matrix is None:
transform_matrix = self.shift()
else:
transform_matrix = tf.tensordot(transform_matrix, self.shift(), axes=1)
if do_zoom:
if transform_matrix is None:
transform_matrix = self.zoom()
else:
transform_matrix = tf.tensordot(transform_matrix, self.zoom(), axes=1)
if do_shear:
if transform_matrix is None:
transform_matrix = self.shear()
else:
transform_matrix = tf.tensordot(transform_matrix, self.shear(), axes=1)
self.transform_matrix = transform_matrix
if self.flip_left_right_boolean:
self.do_flip_lr_tensor = self.flip()
if self.flip_up_down_boolean:
self.do_flip_ud_tensor = self.flip()
def call(self, inputs):
inputs = tf.convert_to_tensor(inputs)
return inputs * tf.sigmoid(inputs)
def quantization_offset(self):
if self._quantization_offset is None:
return None
return tf.convert_to_tensor(self._quantization_offset)
def evaluate_multiplesentence(inp_sentence,
testing,
calc_prob=False,
cal_attention=False):
no_of_inp_sentence = len(inp_sentence)
start_token = [tokenizer_src.vocab_size]
end_token = [tokenizer_src.vocab_size + 1]
if testing:
inp_sentence = [tokenizer_src.encode(x) for x in inp_sentence]
else:
inp_sentence = [tokenizer_src.encode(x[0]) for x in inp_sentence]
inp_sentence = [start_token + x + end_token for x in inp_sentence]
source_endlist = []
if cal_attention:
for i in inp_sentence:
if len(i) <= MAX_LENGTH:
source_endlist.append(len(i))
else:
source_endlist.append(MAX_LENGTH)
inp_sentence = add_post_pad(inp_sentence, testing)
encoder_input = tf.convert_to_tensor(inp_sentence)
dec_input = [[tokenizer_tar.vocab_size] for i in range(no_of_inp_sentence)]
output = tf.convert_to_tensor(dec_input)
dec_input = tf.convert_to_tensor(dec_input)
hidden = (tf.zeros(
(no_of_inp_sentence, units)), tf.zeros((no_of_inp_sentence, units)))
enc_out, enc_hidden = encoder(encoder_input, hidden)
enc_mask = (tf.cast(tf.math.equal(encoder_input, 0), tf.float32) * -1e9)
enc_mask = tf.expand_dims(enc_mask, -1)
dec_hidden = enc_hidden
attention_weigths_np = ""
if cal_attention:
attention_weigths_np = np.zeros(
(MAX_LENGTH, no_of_inp_sentence, MAX_LENGTH))
output_prob = ""
if calc_prob:
output_prob = tf.cast(
tf.convert_to_tensor([[1] for i in range(no_of_inp_sentence)]),
tf.float32)
for t in range(MAX_LENGTH):
predictions, dec_hidden, attention_weights = decoder(
dec_input, dec_hidden, enc_out, enc_mask)
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
dec_input = tf.reshape(predicted_id, (predicted_id.shape[0], -1))
output = tf.concat([output, dec_input], axis=-1)
if calc_prob:
predicted_prob = tf.nn.softmax(predictions)
predicted_prob = tf.reshape(tf.reduce_max(predicted_prob, axis=-1),
(-1, 1))
output_prob = tf.concat([output_prob, predicted_prob], axis=-1)
if cal_attention:
attention_weigths_np[t] = attention_weights.numpy().reshape(
-1, MAX_LENGTH)
return output, attention_weigths_np, output_prob, source_endlist
def regularizer(tensor):
with tf.name_scope(scope, default_name='l2_regularizer', values=[tensor]):
l2_weight = tf.convert_to_tensor(weight_decay,
dtype=tensor.dtype.base_dtype,
name='weight_decay')
return tf.multiply(l2_weight, tf.nn.l2_loss(tensor), name='value')
def transform(
images: TensorLike,
transforms: TensorLike,
interpolation: str = "nearest",
fill_mode: str = "constant",
output_shape: Optional[list] = None,
name: Optional[str] = None,
fill_value: TensorLike = 0.0,
) -> tf.Tensor:
"""Applies the given transform(s) to the image(s).
Args:
images: A tensor of shape (num_images, num_rows, num_columns,
num_channels) (NHWC), (num_rows, num_columns, num_channels) (HWC), or
(num_rows, num_columns) (HW).
transforms: Projective transform matrix/matrices. A vector of length 8 or
tensor of size N x 8. If one row of transforms is
[a0, a1, a2, b0, b1, b2, c0, c1], then it maps the *output* point
`(x, y)` to a transformed *input* point
`(x', y') = ((a0 x + a1 y + a2) / k, (b0 x + b1 y + b2) / k)`,
where `k = c0 x + c1 y + 1`. The transforms are *inverted* compared to
the transform mapping input points to output points. Note that
gradients are not backpropagated into transformation parameters.
interpolation: Interpolation mode.
Supported values: "nearest", "bilinear".
fill_mode: Points outside the boundaries of the input are filled according
to the given mode (one of `{'constant', 'reflect', 'wrap', 'nearest'}`).
- *reflect*: `(d c b a | a b c d | d c b a)`
The input is extended by reflecting about the edge of the last pixel.
- *constant*: `(k k k k | a b c d | k k k k)`
The input is extended by filling all values beyond the edge with the
same constant value k = 0.
- *wrap*: `(a b c d | a b c d | a b c d)`
The input is extended by wrapping around to the opposite edge.
- *nearest*: `(a a a a | a b c d | d d d d)`
The input is extended by the nearest pixel.
fill_value: a float represents the value to be filled outside the
boundaries when `fill_mode` is "constant".
output_shape: Output dimesion after the transform, [height, width].
If None, output is the same size as input image.
name: The name of the op.
Returns:
Image(s) with the same type and shape as `images`, with the given
transform(s) applied. Transformed coordinates outside of the input image
will be filled with zeros.
Raises:
TypeError: If `image` is an invalid type.
ValueError: If output shape is not 1-D int32 Tensor.
"""
with tf.name_scope(name or "transform"):
image_or_images = tf.convert_to_tensor(images, name="images")
transform_or_transforms = tf.convert_to_tensor(transforms,
name="transforms",
dtype=tf.dtypes.float32)
if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
raise TypeError("Invalid dtype %s." % image_or_images.dtype)
images = img_utils.to_4D_image(image_or_images)
original_ndims = img_utils.get_ndims(image_or_images)
if output_shape is None:
output_shape = tf.shape(images)[1:3]
output_shape = tf.convert_to_tensor(output_shape,
tf.dtypes.int32,
name="output_shape")
if not output_shape.get_shape().is_compatible_with([2]):
raise ValueError(
"output_shape must be a 1-D Tensor of 2 elements: "
"new_height, new_width")
if len(transform_or_transforms.get_shape()) == 1:
transforms = transform_or_transforms[None]
elif transform_or_transforms.get_shape().ndims is None:
raise ValueError("transforms rank must be statically known")
elif len(transform_or_transforms.get_shape()) == 2:
transforms = transform_or_transforms
else:
transforms = transform_or_transforms
raise ValueError(
"transforms should have rank 1 or 2, but got rank %d" %
len(transforms.get_shape()))
if LooseVersion(tf.__version__) >= LooseVersion("2.4.0"):
fill_value = tf.convert_to_tensor(fill_value,
dtype=tf.float32,
name="fill_value")
output = tf.raw_ops.ImageProjectiveTransformV3(
images=images,
transforms=transforms,
output_shape=output_shape,
interpolation=interpolation.upper(),
fill_mode=fill_mode.upper(),
fill_value=fill_value,
)
else:
fill_mode = fill_mode.upper()
# TODO(WindQAQ): Get rid of the check once we drop TensorFlow < 2.4 support.
if fill_mode == "CONSTANT":
warnings.warn(
"fill_value is not supported and is always 0 for TensorFlow < 2.4.0."
)
if fill_mode == "NEAREST":
raise ValueError(
"NEAREST fill_mode is not supported for TensorFlow < 2.4.0."
)
output = tf.raw_ops.ImageProjectiveTransformV2(
images=images,
transforms=transforms,
output_shape=output_shape,
interpolation=interpolation.upper(),
fill_mode=fill_mode,
)
return img_utils.from_4D_image(output, original_ndims)
def expand_tile(value, size):
"""Add a new axis of given size."""
value = tf.convert_to_tensor(value=value, name='value')
ndims = value.shape.ndims
return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims)
def foo(x, y, z=3):
# Since we don't wrap this as a tf.function, we need to do some
# tf.convert_to_tensor(...) in order to ensure we have TensorFlow types.
x = tf.convert_to_tensor(x)
y = tf.convert_to_tensor(y)
return (x + y, tf.convert_to_tensor(z))
def inference(self, inputs, name):
inputs = tf.convert_to_tensor(inputs)
with tf.variable_scope(name):
# reshape from inputs
with tf.variable_scope('reshape'):
outputs = tf.layers.dense(
inputs,
self.depths[0] * self.s_size * self.s_size,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args[self.name]['weight_decay']))
outputs = tf.reshape(
outputs, [-1, self.s_size, self.s_size, self.depths[0]])
outputs = tf.nn.leaky_relu(tf.layers.batch_normalization(
outputs, training=True),
name='outputs')
outputs = tf.clip_by_value(outputs, -5, 5)
# deconvolution (transpose of convolution) x 4
with tf.variable_scope('deconv1'):
outputs = tf.layers.conv2d_transpose(
outputs,
self.depths[1], [5, 5],
strides=(2, 2),
padding='SAME',
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args[self.name]['weight_decay']))
outputs = tf.nn.leaky_relu(tf.layers.batch_normalization(
outputs, training=True),
name='outputs')
outputs = tf.clip_by_value(outputs, -5, 5)
with tf.variable_scope('deconv2'):
outputs = tf.layers.conv2d_transpose(
outputs,
self.depths[2], [5, 5],
strides=(2, 2),
padding='SAME',
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args[self.name]['weight_decay']))
outputs = tf.nn.leaky_relu(tf.layers.batch_normalization(
outputs, training=True),
name='outputs')
outputs = tf.clip_by_value(outputs, -5, 5)
with tf.variable_scope('deconv3'):
outputs = tf.layers.conv2d_transpose(
outputs,
self.depths[3], [5, 5],
strides=(2, 2),
padding='SAME',
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args[self.name]['weight_decay']))
outputs = tf.nn.leaky_relu(tf.layers.batch_normalization(
outputs, training=True),
name='outputs')
outputs = tf.clip_by_value(outputs, -5, 5)
with tf.variable_scope('deconv4'):
outputs = tf.layers.conv2d_transpose(
outputs,
self.depths[4], [5, 5],
strides=(2, 2),
padding='SAME',
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args[self.name]['weight_decay']))
# output images
with tf.variable_scope('tanh'):
outputs = tf.tanh(outputs, name='outputs')
return outputs
def _compute_tf(
self,
x: "tf.Tensor",
x_init: "tf.Tensor",
y: "tf.Tensor",
mask: "tf.Tensor",
eps: Union[int, float, np.ndarray],
eps_step: Union[int, float, np.ndarray],
random_init: bool,
) -> "tf.Tensor":
"""
Compute adversarial examples for one iteration.
:param x: Current adversarial examples.
:param x_init: An array with the original inputs.
:param y: Target values (class labels) one-hot-encoded of shape `(nb_samples, nb_classes)` or indices of shape
(nb_samples,). Only provide this parameter if you'd like to use true labels when crafting adversarial
samples. Otherwise, model predictions are used as labels to avoid the "label leaking" effect
(explained in this paper: https://arxiv.org/abs/1611.01236). Default is `None`.
:param mask: An array with a mask broadcastable to input `x` defining where to apply adversarial perturbations.
Shape needs to be broadcastable to the shape of x and can also be of the same shape as `x`. Any
features for which the mask is zero will not be adversarially perturbed.
:param eps: Maximum perturbation that the attacker can introduce.
:param eps_step: Attack step size (input variation) at each iteration.
:param random_init: Random initialisation within the epsilon ball. For random_init=False starting at the
original input.
:return: Adversarial examples.
"""
import tensorflow as tf # lgtm [py/repeated-import]
if random_init:
n = x.shape[0]
m = np.prod(x.shape[1:]).item()
random_perturbation = random_sphere(n, m, eps, self.norm).reshape(x.shape).astype(ART_NUMPY_DTYPE)
random_perturbation = tf.convert_to_tensor(random_perturbation)
if mask is not None:
random_perturbation = random_perturbation * mask
x_adv = x + random_perturbation
if self.estimator.clip_values is not None:
clip_min, clip_max = self.estimator.clip_values
x_adv = tf.clip_by_value(x_adv, clip_min, clip_max)
else:
x_adv = x
# Get perturbation
perturbation = self._compute_perturbation(x_adv, y, mask)
# Apply perturbation and clip
x_adv = self._apply_perturbation(x_adv, perturbation, eps_step)
# Do projection
perturbation = self._projection(x_adv - x_init, eps, self.norm)
# Recompute x_adv
x_adv = tf.add(perturbation, x_init)
return x_adv
def process_image(image):
image_tensor = tf.convert_to_tensor(image)
image_tensor = tf.image.resize(image_tensor,
tf.Variable([224, 224], tf.int32))
image_tensor /= 255
return image_tensor.numpy()
def TiSAS_multihead_attention(
self,
queries,
keys,
key_length,
query_length,
t_querys,
t_keys,
t_querys_length,
t_keys_length,
num_units=None,
num_heads=8,
dropout_rate=0,
is_training=True,
scope="multihead_attention",
reuse=None,
):
'''Applies multihead attention.
Args:
queries: A 3d tensor with shape of [N, T_q, C_q].
queries_length: A 1d tensor with shape of [N].
keys: A 3d tensor with shape of [N, T_k, C_k].
keys_length: A 1d tensor with shape of [N].
num_units: A scalar. Attention size.
dropout_rate: A floating point number.
is_training: Boolean. Controller of mechanism for dropout.
num_heads: An int. Number of heads.
scope: Optional scope for `variable_scope`.
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns
A 3d tensor with shape of (N, T_q, C)
'''
t_querys = tf.stack([t_querys] * t_keys_length, axis=2)
t_keys = tf.stack([t_keys] * t_querys_length, axis=1)
# decay = tf.relu(time_w * tf.log((t_querys - tf.transpose(t_keys))+1)+time_b)
interval = tf.log(tf.add(tf.abs(tf.subtract(t_querys, t_keys)), 1))
# Linear projections, C = # dim or column, T_x = # vectors or actions
Q = tf.layers.dense(queries, num_units,
activation=tf.nn.relu) # (N, T_q, C)
#Q = tf.layers.dropout(Q, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
K = tf.layers.dense(keys, num_units,
activation=tf.nn.relu) # (N, T_k, C)
#K = tf.layers.dropout(K, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
V = tf.layers.dense(keys, num_units,
activation=tf.nn.relu) # (N, T_k, C)
#V = tf.layers.dropout(V, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
with tf.variable_scope(scope, reuse=reuse):
# Set the fall back option for num_units
if num_units is None:
num_units = queries.get_shape().as_list()[-1]
# Split and concat
Q_ = tf.concat(tf.split(Q, num_heads, axis=2),
axis=0) # (h*N, T_q, C/h)
K_ = tf.concat(tf.split(K, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
V_ = tf.concat(tf.split(V, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
interval_ = tf.concat([interval] * num_heads,
axis=0) # (h*N, T_k, C/h)
#decay_gate_ = tf.layers.dropout(decay_gate_, rate=dropout_rate,
#training=tf.convert_to_tensor(is_training))
# Multiplication
# query-key score matrix
# each big score matrix is then split into h score matrix with same size
# w.r.t. different part of the feature
outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1]),
name='3') # (h*N, T_q, T_k)
outputs += interval_
# Scale
outputs = outputs / (K_.get_shape().as_list()[-1]**0.5)
# Key Masking
#key_masks = tf.sign(tf.abs(tf.reduce_sum(keys, axis=-1)))# (N, T_k)
key_masks = tf.sequence_mask(key_length,
tf.shape(keys)[1]) # (N, T_k)
key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
key_masks = tf.tile(tf.expand_dims(
key_masks, 1), [1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(outputs) * (-2**32 + 1)
#outputs = tf.where(tf.equal(key_masks, 0), outputs, paddings) # (h*N, T_q, T_k)
outputs = tf.where(key_masks, outputs, paddings) # (h*N, T_q, T_k)
# Causality = Future blinding: No use, removed
# Activation
outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
'''
# Query Masking
query_masks = tf.sign(tf.abs(tf.reduce_sum(queries, axis=-1))) # (N, T_q)
query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
outputs *= query_masks # broadcasting. (N, T_q, C)
# Attention vector
att_vec = outputs
# Dropouts
outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
# Weighted sum
outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2) # (N, T_q, C)
# Residual connection
outputs += queries
# Normalize
outputs = self.normalize(outputs) # (N, T_q, C)
'''
# Query Masking
#query_masks = tf.sign(tf.abs(tf.reduce_sum(queries, axis=-1))) # (N, T_q)
query_masks = tf.sequence_mask(query_length,
tf.shape(queries)[1],
dtype=tf.float32) # (N, T_q)
query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
query_masks = tf.tile(tf.expand_dims(query_masks, -1),
[1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
outputs *= query_masks # broadcasting. (N, T_q, C)
print(outputs.shape.as_list())
print(query_masks.shape.as_list())
# Attention vector
#########Tom Sun
att_vec = outputs
# Dropouts
outputs = tf.layers.dropout(
outputs,
rate=dropout_rate,
training=tf.convert_to_tensor(is_training))
# Weighted sum
outputs = tf.matmul(outputs, V_, name='4') # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0),
axis=2) # (N, T_q, C)
outputs = outputs
# Residual connection
outputs += queries
# Normalize
outputs = self.normalize(outputs) # (N, T_q, C)
return outputs, att_vec
def denormalize(self, object, inputs):
inputs = tf.convert_to_tensor(inputs)
y = (self.scale * inputs) + self.loc
return y, y
def Bundle_Adjustment_optimization(params):
'''
Using Bundle Adjustment optimize SMPL parameters with tensorflow-gpu
:param hmr_dict:
:param data_dict:
:return: results
'''
start_time = time.time()
start_time_total = time.time()
Util = Utility()
Util.read_utility_parameters(params)
smpl_model = SMPL(Util.SMPL_COCO_PATH, Util.SMPL_NORMAL_PATH)
j3dss, success = Util.load_pose_pkl()
hmr_dict, data_dict = Util.load_hmr_data()
hmr_thetas = hmr_dict["hmr_thetas"]
hmr_betas = hmr_dict["hmr_betas"]
hmr_trans = hmr_dict["hmr_trans"]
hmr_cams = hmr_dict["hmr_cams"]
hmr_joint3ds = hmr_dict["hmr_joint3ds"]
j2ds = data_dict["j2ds"]
confs = data_dict["confs"]
j2ds_face = data_dict["j2ds_face"]
confs_face = data_dict["confs_face"]
j2ds_head = data_dict["j2ds_head"]
confs_head = data_dict["confs_head"]
j2ds_foot = data_dict["j2ds_foot"]
confs_foot = data_dict["confs_foot"]
imgs = data_dict["imgs"]
masks = data_dict["masks"]
Util.img_width = imgs[0].shape[1]
Util.img_height = imgs[0].shape[0]
Util.img_widthheight = int("1" + "%04d" % Util.img_width + "%04d" % Util.img_height)
frame_num = len(j2ds)
for ind in range(frame_num):
hmr_theta = hmr_thetas[ind, :].squeeze()
# hmr_shape = hmr_betas[ind, :].squeeze()
# hmr_tran = hmr_trans[ind, :].squeeze()
# hmr_cam = hmr_cams[0, :].squeeze()
hmr_joint3d = hmr_joint3ds[ind, :, :]
######### Arm Correction #########
# if Util.pedestrian_constraint == True and success == True:
# prej3d = j3dss[ind]
# if abs(prej3d[2, 2] - prej3d[7, 2]) > 0.1:
# print("leg_error>0.1")
# if prej3d[2, 2] < prej3d[7, 2]:
# hmr_thetas[ind][51] = 0.8
# hmr_theta[52] = 1e-8
# hmr_theta[53] = 1.0
# hmr_theta[58] = 1e-8
# forward_arm = "left"
# else:
# hmr_theta[48] = 0.8
# hmr_theta[49] = 1e-8
# hmr_theta[50] = -1.0
# hmr_theta[55] = 1e-8
# forward_arm = "right"
if Util.pedestrian_constraint == True:
if abs(hmr_joint3ds[ind, 0, 2] - hmr_joint3ds[ind, 5, 2]) > 0.1:
print("leg_error>0.1")
if hmr_joint3ds[ind, 0, 2] < hmr_joint3ds[ind, 5, 2]:
hmr_thetas[ind, 51] = 0.8
hmr_thetas[ind, 52] = 1e-8
hmr_thetas[ind, 53] = 1.0
hmr_thetas[ind, 58] = 1e-8
forward_arm = "left"
else:
hmr_thetas[ind, 48] = 0.8
hmr_thetas[ind, 49] = 1e-8
hmr_thetas[ind, 50] = -1.0
hmr_thetas[ind, 55] = 1e-8
forward_arm = "right"
initial_param, pose_mean, pose_covariance = Util.load_initial_param()
pose_mean = tf.constant(pose_mean, dtype=tf.float32)
pose_covariance = tf.constant(pose_covariance, dtype=tf.float32)
###batch input
batch_size = frame_num / Util.batch_num
for ind_batch in range(Util.batch_num):
ind_start = ind_batch * batch_size
ind_end = (ind_batch+1) * batch_size
print(ind_start, ind_end)
param_shapes = tf.Variable(hmr_betas[ind_start:ind_end,:].reshape([-1, 10]), dtype=tf.float32)
param_rots = tf.Variable(hmr_thetas[ind_start:ind_end, :3].reshape([-1, 3]), dtype=tf.float32)
param_poses = tf.Variable(hmr_thetas[ind_start:ind_end, 3:72].reshape([-1, 69]), dtype=tf.float32)
param_trans = tf.Variable(hmr_trans[ind_start:ind_end,:].reshape([-1, 3]), dtype=tf.float32)
initial_param_tf = tf.concat([param_shapes, param_rots, param_poses, param_trans], axis=1) ## N * (72+10+3)
hmr_cam = hmr_cams[0, :].squeeze()
cam = Perspective_Camera(hmr_cam[0], hmr_cam[0], hmr_cam[1],
hmr_cam[2], np.zeros(3), np.zeros(3))
j3ds, v, j3dsplus = smpl_model.get_3d_joints(initial_param_tf, Util.SMPL_JOINT_IDS)
#### divide into different body parts
j3ds_body = j3ds[:, 2:, :]
j3ds_head = j3ds[:, 14:16, :]
j3ds_foot = j3ds[:, :2, :]
j3ds_face = j3dsplus[:, 14:19, :]
j3ds_body = tf.reshape(j3ds_body, [-1, 3]) ## (N*12) * 3
j3ds_head = tf.reshape(j3ds_head, [-1, 3]) ## (N*2) * 3
j3ds_foot = tf.reshape(j3ds_foot, [-1, 3]) ## (N*2) * 3
j3ds_face = tf.reshape(j3ds_face, [-1, 3]) ## (N*5) * 3
j2ds_body_est = cam.project(tf.squeeze(j3ds_body)) ## (N*14) * 2
j2ds_head_est = cam.project(tf.squeeze(j3ds_head)) ## (N*2) * 2
j2ds_foot_est = cam.project(tf.squeeze(j3ds_foot)) ## (N*2) * 2
j2ds_face_est = cam.project(tf.squeeze(j3ds_face)) ## (N*5) * 2
v = tf.reshape(v, [-1, 3]) ## (N*6890) * 3
verts_est_mask = cam.project(tf.squeeze(v)) ## (N*6890) * 2
verts_est = cam.project(tf.squeeze(v)) ## (N*6890) * 2
# TODO convert the loss function into batch input
objs = {}
j2ds_batch = j2ds[ind_start:ind_end, :, :].reshape([-1, 2]) ## (N*14) * 2
confs_batch = confs[ind_start:ind_end, :].reshape(-1) ## N*14
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0])
base_weights = np.tile(base_weights, batch_size) ## N*14
weights = confs_batch * base_weights ## N*14
weights = tf.constant(weights, dtype=tf.float32) ## N*14
objs['J2D_Loss'] = Util.J2D_Loss * tf.reduce_sum(weights * tf.reduce_sum(tf.square(j2ds_body_est - j2ds_batch), 1))
j2ds_face_batch = j2ds_face[ind_start:ind_end, :, :].reshape([-1, 2]) ## (N*5) * 2
confs_face_batch = confs_face[ind_start:ind_end, :].reshape(-1) ## N*5
base_weights_face = np.array(
[1.0, 1.0, 1.0, 1.0, 1.0])
base_weights_face = np.tile(base_weights_face, batch_size) ## N*5
weights_face = confs_face_batch * base_weights_face
weights_face = tf.constant(weights_face, dtype=tf.float32)
objs['J2D_face_Loss'] = Util.J2D_face_Loss * tf.reduce_sum(
weights_face * tf.reduce_sum(tf.square(j2ds_face_est - j2ds_face_batch), 1))
j2ds_head_batch = j2ds_head[ind_start:ind_end, :, :].reshape([-1, 2]) ## (N*2) * 2
confs_head_batch = confs_head[ind_start:ind_end, :].reshape(-1) ## N*2
base_weights_head = np.array(
[1.0, 1.0])
base_weights_head = np.tile(base_weights_head, batch_size) ## N*2
weights_head = confs_head_batch * base_weights_head
weights_head = tf.constant(weights_head, dtype=tf.float32)
objs['J2D_head_Loss'] = Util.J2D_head_Loss * tf.reduce_sum(
weights_head * tf.reduce_sum(tf.square(j2ds_head_batch - j2ds_head_est), 1))
j2ds_foot_batch = j2ds_foot[ind_start:ind_end, :, :].reshape([-1, 2]) ## (N*2) * 2
confs_foot_batch = confs_foot[ind_start:ind_end, :].reshape(-1) ## N*2
base_weights_foot = np.array(
[1.0, 1.0])
base_weights_foot = np.tile(base_weights_foot, batch_size) ## N*2
weights_foot = confs_foot_batch * base_weights_foot ## N*2
weights_foot = tf.constant(weights_foot, dtype=tf.float32)
objs['J2D_foot_Loss'] = Util.J2D_foot_Loss * tf.reduce_sum(
weights_foot * tf.reduce_sum(tf.square(j2ds_foot_batch - j2ds_foot_est), 1))
for i in range(batch_size):
pose_diff = tf.reshape(param_poses[i, :] - pose_mean, [1, -1])
if i == 0:
objs['Prior_Loss'] = 1.0 * tf.squeeze(
tf.matmul(tf.matmul(pose_diff, pose_covariance), tf.transpose(pose_diff)))
else:
objs['Prior_Loss'] = objs['Prior_Loss'] + 1.0 * tf.squeeze(
tf.matmul(tf.matmul(pose_diff, pose_covariance), tf.transpose(pose_diff)))
objs['Prior_Shape'] = 5.0 * tf.reduce_sum(tf.square(param_shapes))
w1 = np.array([1.04 * 2.0, 1.04 * 2.0, 5.4 * 2.0, 5.4 * 2.0])
w1 = tf.constant(w1, dtype=tf.float32)
# objs["angle_elbow_knee"] = 0.008 * tf.reduce_sum(w1 * [
# tf.exp(param_poses[:, 52]), tf.exp(-param_poses[:, 55]),
# tf.exp(-param_poses[:, 9]), tf.exp(-param_poses[:, 12])])
objs["angle_elbow_knee"] = 0.08 * tf.reduce_sum(w1[0] *
tf.exp(param_poses[:, 52]) + w1[1] *
tf.exp(-param_poses[:, 55]) + w1[2] *
tf.exp(-param_poses[:, 9]) + w1[3] *
tf.exp(-param_poses[:, 12]))
# TODO add a function that deal with masks with batch
tmp_batch = []
for i in range(ind_start, ind_end):
verts2dsilhouette = algorithms.verts_to_silhouette_tf(verts_est_mask, masks[i].shape[1], masks[i].shape[0])
tmp_batch.append(verts2dsilhouette)
verts2dsilhouette_batch = tf.convert_to_tensor(tmp_batch)
mask = np.array(masks[ind_start:ind_end])
masks_tf = tf.cast(tf.convert_to_tensor(mask), dtype=tf.float32)
objs['mask'] = Util.mask * tf.reduce_sum(
verts2dsilhouette_batch / 255.0 * (255.0 - masks_tf) / 255.0
+ (255.0 - verts2dsilhouette_batch) / 255.0 * masks_tf / 255.0)
# TODO try L1, L2 or other penalty function
param_pose_full = tf.concat([param_rots, param_poses], axis=1) ## N * 72
objs['hmr_constraint'] = Util.hmr_constraint * tf.reduce_sum(
tf.square(tf.squeeze(param_pose_full) - hmr_thetas[ind_start:ind_end,:]))
objs['hmr_hands_constraint'] = Util.hmr_hands_constraint * tf.reduce_sum(
tf.square(tf.squeeze(param_pose_full)[:, 21] - hmr_thetas[ind_start:ind_end, 21])
+ tf.square(tf.squeeze(param_pose_full)[:, 23] - hmr_thetas[ind_start:ind_end, 23])
+ tf.square(tf.squeeze(param_pose_full)[:, 20] - hmr_thetas[ind_start:ind_end, 20])
+ tf.square(tf.squeeze(param_pose_full)[:, 22] - hmr_thetas[ind_start:ind_end, 22]))
# w_temporal = [0.5, 0.5, 1.0, 1.5, 2.5, 2.5, 1.5, 1.0, 1.0, 1.5, 2.5, 2.5, 1.5, 1.0, 7.0, 7.0]
# for i in range(frame_num-1):
# j3d_old = j3ds[i, :, :]
# j3d = j3ds[i + 1, :, :]
# j3d_old_tmp = tf.reshape(j3d_old, [-1, 3]) ## (N*16) * 3
# j2d_old = cam.project(tf.squeeze(j3d_old_tmp)) ## (N*16) * 2
# j3d_tmp = tf.reshape(j3d, [-1, 3]) ## (N*16) * 3
# j2d = cam.project(tf.squeeze(j3d_tmp)) ## (N*16) * 2
# param_pose_old = param_poses[i, :]
# param_pose = param_poses[i+1, :]
# if i == 0:
# objs['temporal3d'] = Util.temporal3d * tf.reduce_sum(
# w_temporal * tf.reduce_sum(tf.square(j3d - j3d_old), 1))
# objs['temporal2d'] = Util.temporal2d * tf.reduce_sum(
# w_temporal * tf.reduce_sum(tf.square(j2d - j2d_old), 1))
# objs['temporal_pose'] = Util.temporal_pose * tf.reduce_sum(
# tf.square(param_pose_old - param_pose))
# else:
# objs['temporal3d'] = objs['temporal3d'] + Util.temporal3d * tf.reduce_sum(
# w_temporal * tf.reduce_sum(tf.square(j3d - j3d_old), 1))
# objs['temporal2d'] = objs['temporal2d'] + Util.temporal2d * tf.reduce_sum(
# w_temporal * tf.reduce_sum(tf.square(j2d - j2d_old), 1))
# objs['temporal_pose'] = objs['temporal_pose'] + Util.temporal_pose * tf.reduce_sum(
# tf.square(param_pose_old - param_pose))
# TODO add optical flow constraint
# body_idx = np.array(body_parsing_idx[0]).squeeze()
# body_idx = body_idx.reshape([-1, 1]).astype(np.int64)
# verts_est_body = tf.gather_nd(verts_est, body_idx)
# optical_ratio = 0.0
# objs['dense_optflow'] = util.params["LR_parameters"]["dense_optflow"] * tf.reduce_sum(tf.square(
# verts_est_body - verts_body_old))
# optimization process
loss = tf.reduce_sum(objs.values())
duration = time.time() - start_time
print("pre-processing time is %f" % duration)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
optimizer = scipy_pt(loss=loss,
var_list=[param_shapes, param_rots, param_trans, param_poses, cam.cx, cam.cy],
options={'eps': 1e-6, 'ftol': 1e-6, 'maxiter': 10000, 'disp': True})
print(">>>>>>>>>>>>>start to optimize<<<<<<<<<<<")
start_time = time.time()
optimizer.minimize(sess)
duration = time.time() - start_time
print("minimize is %f" % duration)
start_time = time.time()
poses_final, betas_final, trans_final, cam_cx, cam_cy, v_final, verts_est_final, j3ds_final, _objs = sess.run([tf.concat([param_rots, param_poses], axis=1),
param_shapes, param_trans, cam.cx, cam.cy, v, verts_est, j3ds, objs])
v_final = v_final.reshape([batch_size, 6890, 3])
duration = time.time() - start_time
print("run time is %f" % duration)
start_time = time.time()
cam_for_save = np.array([hmr_cam[0], cam_cx, cam_cy, np.zeros(3)])
### no sense
LR_cameras = []
for i in range(ind_start, ind_end):
LR_cameras.append(cam_for_save)
#############
camera = render.camera(cam_for_save[0], cam_for_save[1], cam_for_save[2], cam_for_save[3], Util.img_widthheight)
output_path = Util.hmr_path + Util.output_path
if not os.path.exists(output_path):
os.makedirs(output_path)
if not os.path.exists(Util.hmr_path + "output_mask"):
os.makedirs(Util.hmr_path + "output_mask")
videowriter = []
for i, ind in enumerate(range(ind_start, ind_end)):
print(">>>>>>>>>>>>>>>>>>>>>>%d index frame<<<<<<<<<<<<<<<<<<<<<<" % ind)
if Util.mode == "full":
# smpl = smpl_np.SMPLModel('./smpl/models/basicmodel_m_lbs_10_207_0_v1.0.0.pkl')
# template = np.load(Util.texture_path + "template.npy")
# smpl.set_template(template)
# v = smpl.get_verts(poses_final[i, :], betas_final[i, :], trans_final[i, :])
# texture_vt = np.load(Util.texture_path + "vt.npy")
# texture_img = cv2.imread(Util.texture_path + "../../output_nonrigid/texture.png")
# img_result_texture = camera.render_texture(v, texture_img, texture_vt)
# cv2.imwrite(output_path + "/hmr_optimization_texture_%04d.png" % ind, img_result_texture)
# img_bg = cv2.resize(imgs[ind], (Util.img_width, Util.img_height))
# img_result_texture_bg = camera.render_texture_imgbg(img_result_texture, img_bg)
# cv2.imwrite(output_path + "/texture_bg_%04d.png" % ind,
# img_result_texture_bg)
#
# img_result_naked = camera.render_naked(v, imgs[ind])
# img_result_naked = img_result_naked[:, :, :3]
# cv2.imwrite(output_path + "/hmr_optimization_%04d.png" % ind, img_result_naked)
# bg = np.ones_like(imgs[ind]).astype(np.uint8) * 255
# img_result_naked1 = camera.render_naked(v, bg)
# cv2.imwrite(output_path + "/hmr_optimization_naked_%04d.png" % ind, img_result_naked1)
# img_result_naked_rotation = camera.render_naked_rotation(v, 90, imgs[ind])
# cv2.imwrite(output_path + "/hmr_optimization_rotation_%04d.png" % ind,
# img_result_naked_rotation)
res = {'pose': poses_final[i, :], 'betas': betas_final[i, :], 'trans': trans_final[i, :],
'cam': cam_for_save, 'j3ds': j3ds_final[i, :]}
with open(output_path + "/hmr_optimization_pose_%04d.pkl" % ind, 'wb') as fout:
pkl.dump(res, fout)
# for z in range(len(verts_est_final)):
# if int(verts_est_final[z][0]) > masks[ind].shape[0] - 1:
# verts_est_final[z][0] = masks[ind].shape[0] - 1
# if int(verts_est_final[z][1]) > masks[ind].shape[1] - 1:
# verts_est_final[z][1] = masks[ind].shape[1] - 1
# (masks[ind])[int(verts_est_final[z][0]), int(verts_est_final[z][1])] = 127
# cv2.imwrite(Util.hmr_path + "output_mask/%04d.png" % ind, masks[ind])
if Util.mode == "pose":
# img_result_naked = camera.render_naked(v_final[i, :, :], imgs[ind])
# img_result_naked = img_result_naked[:, :, :3]
# cv2.imwrite(output_path + "/hmr_optimization_%04d.png" % ind, img_result_naked)
# bg = np.ones_like(imgs[ind]).astype(np.uint8) * 255
# img_result_naked1 = camera.render_naked(v_final[i, :, :], bg)
# cv2.imwrite(output_path + "/hmr_optimization_naked_%04d.png" % ind, img_result_naked1)
# img_result_naked_rotation = camera.render_naked_rotation(v_final[i, :, :], 90, imgs[ind])
# cv2.imwrite(output_path + "/hmr_optimization_rotation_%04d.png" % ind,
# img_result_naked_rotation)
res = {'pose': poses_final[i, :], 'betas': betas_final[i, :], 'trans': trans_final[i, :],
'cam': cam_for_save, 'j3ds': j3ds_final[i, :]}
with open(output_path + "/hmr_optimization_pose_%04d.pkl" % ind, 'wb') as fout:
pkl.dump(res, fout)
# for z in range(len(verts_est_final)):
# if int(verts_est_final[z][0]) > masks[ind].shape[0] - 1:
# verts_est_final[z][0] = masks[ind].shape[0] - 1
# if int(verts_est_final[z][1]) > masks[ind].shape[1] - 1:
# verts_est_final[z][1] = masks[ind].shape[1] - 1
# (masks[ind])[int(verts_est_final[z][0]), int(verts_est_final[z][1])] = 127
# cv2.imwrite(Util.hmr_path + "output_mask/%04d.png" % ind, masks[ind])
for name in _objs:
print("the %s loss is %f" % (name, _objs[name]))
if Util.video is True:
fps = 15
size = (imgs[0].shape[1], imgs[0].shape[0])
video_path = output_path + "/texture.mp4"
videowriter = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc('D', 'I', 'V', 'X'), fps, size)
if Util.mode == "full":
for i in range(batch_size*Util.batch_num):
img = cv2.imread(output_path + "/texture_bg_%04d.png" % ind)
videowriter.write(img)
if Util.mode == "pose":
for i in range(batch_size*Util.batch_num):
img = cv2.imread(output_path + "/hmr_optimization_%04d.png" % ind)
videowriter.write(img)
util_func.save_pkl_to_csv(output_path)
util_func.save_pkl_to_npy(output_path)
duration = time.time() - start_time
print("post-processing time is %f" % duration)
duration = time.time() - start_time_total
print("total time is %f" % duration)
video_file = '/home/ch/Videos/carla_vod_1.mp4'
cap = cv2.VideoCapture(video_file)
print('Running inference for {}... '.format(video_file), end='')
while cap.isOpened():
ret, image_np = cap.read()
if ret:
# Things to try:
# Flip horizontally
# image_np = np.fliplr(image_np).copy()
# Convert image to grayscale
#image_np = np.tile(
# np.mean(image_np, 2, keepdims=True), (1, 1, 3)).astype(np.uint8)
input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
detections = detect_fn(input_tensor)
# All outputs are batches tensors.
# Convert to numpy arrays, and take index [0] to remove the batch dimension.
# We're only interested in the first num_detections.
num_detections = int(detections.pop('num_detections'))
detections = {key: value[0, :num_detections].numpy()
for key, value in detections.items()}
detections['num_detections'] = num_detections
# detection_classes should be ints.
detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
label_id_offset = 1
def call(self, inputs):
inputs = tf.convert_to_tensor(inputs)
y = (inputs - self.loc) / self.scale
return y
# Network params
prob = 0.5
num_classes = 10
skip_layers = ['fc8','fc7']
# TF placeholder for graph input and output
keep_prob = tf.placeholder(tf.float32)
## train data
xs, ys = read_and_decode(source_dir, batch_size, is_batch=True)
xt, yt = read_and_decode(target_dir, batch_size, is_batch=True)
X_train = tf.concat([xs, xt],0)
## test data
X_test, Y_test = read_and_decode(target_dir, batch_size, is_batch=False)
X_test = tf.convert_to_tensor(X_test)
Y_test = tf.convert_to_tensor(Y_test)
## model
fc6, fc7, source_logits = alexnet(X_train,num_classes,keep_prob,is_reuse=False)
_,_,logits = alexnet(X_test[:300,...],num_classes,keep_prob,is_reuse=True)
source_fc6 = fc6[:batch_size,...]
target_fc6 = fc6[batch_size:,...]
source_fc7 = fc7[:batch_size,...]
target_fc7 = fc7[batch_size:,...]
## source loss
source_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits = source_logits[:batch_size,...], labels = ys))
## MMD
bandwidths = [2.0, 5.0, 10.0, 20.0, 40.0, 80.0]
loss_fc6,var6,MMD_fc6 = mix_rbf_mmd2_and_ratio(source_fc6, target_fc6,sigmas=bandwidths)
def normalize(self, object, inputs):
inputs = tf.convert_to_tensor(inputs)
y = (inputs - self.loc) / self.scale
return y, y
def run(self):
total_step = 1
mem = Memory()
while Worker.global_episode < args.max_eps:
current_state = self.env.reset()
mem.clear()
ep_reward = 0.
ep_steps = 0
self.ep_loss = 0
time_count = 0
done = False
while not done:
logits, _ = self.local_model(
tf.convert_to_tensor(current_state[None, :],
dtype=tf.float32))
probs = tf.nn.softmax(logits)
action = np.random.choice(self.action_size, p=probs.numpy()[0])
new_state, reward, done, _ = self.env.step(action)
if done:
reward = -1
ep_reward += reward
mem.store(current_state, action, reward)
if time_count == args.update_freq or done:
# Calculate gradient wrt to local model. We do so by tracking the
# variables involved in computing the loss by using tf.GradientTape
with tf.GradientTape() as tape:
total_loss = self.compute_loss(done,
new_state,
mem,
args.gamma)
self.ep_loss += total_loss
# Calculate local gradients
grads = tape.gradient(total_loss, self.local_model.trainable_weights)
# Push local gradients to global model
self.opt.apply_gradients(zip(grads,
self.global_model.trainable_weights))
# Update local model with new weights
self.local_model.set_weights(self.global_model.get_weights())
mem.clear()
time_count = 0
if done: # done and print information
Worker.global_moving_average_reward = \
record(Worker.global_episode, ep_reward, self.worker_idx,
Worker.global_moving_average_reward, self.result_queue,
self.ep_loss, ep_steps)
# We must use a lock to save our model and to print to prevent data races.
if ep_reward > Worker.best_score:
with Worker.save_lock:
print("Saving best model to {}, "
"episode score: {}".format(self.save_dir, ep_reward))
self.global_model.save_weights(
os.path.join(self.save_dir,
'model_{}.h5'.format(self.game_name))
)
Worker.best_score = ep_reward
Worker.global_episode += 1
ep_steps += 1
time_count += 1
current_state = new_state
total_step += 1
self.result_queue.put(None)
def tracker(hp, run, design, frame_name_list, pos_x, pos_y, target_w, target_h,
final_score_sz, filename, image, templates_z, scores, start_frame):
num_frames = np.size(frame_name_list)
# stores tracker's output for evaluation
bboxes = np.zeros((num_frames, 4))
bboxesupper = np.zeros((num_frames, 4))
bboxeslower = np.zeros((num_frames, 4))
scale_factors = hp.scale_step**np.linspace(-np.ceil(hp.scale_num / 2),
np.ceil(hp.scale_num / 2),
hp.scale_num)
# cosine window to penalize large displacements
hann_1d = np.expand_dims(np.hanning(final_score_sz), axis=0)
penalty = np.transpose(hann_1d) * hann_1d
penalty = penalty / np.sum(penalty)
context = design.context * (target_w + target_h)
z_sz = np.sqrt(np.prod((target_w + context) * (target_h + context)))
x_sz = float(design.search_sz) / design.exemplar_sz * z_sz
# thresholds to saturate patches shrinking/growing
min_z = hp.scale_min * z_sz
max_z = hp.scale_max * z_sz
min_x = hp.scale_min * x_sz
max_x = hp.scale_max * x_sz
# run_metadata = tf.RunMetadata()
# run_opts = {
# 'options': tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
# 'run_metadata': run_metadata,
# }
run_opts = {}
# with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
with tf.Session() as sess:
tf.global_variables_initializer().run()
# Coordinate the loading of image files.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# save first frame position (from ground-truth)
bboxes[
0, :] = pos_x - target_w / 2, pos_y - target_h / 2, target_w, target_h
bboxesupper[
0, :] = pos_x - target_w / 2, pos_y - target_h / 2, target_w, target_h / 2
bboxeslower[0, :] = pos_x - target_w / 2, pos_y, target_w, target_h / 2
image_, templates_z_ = sess.run(
[image, templates_z],
feed_dict={
siam.pos_x_ph: pos_x,
siam.pos_y_ph: pos_y,
siam.z_sz_ph: z_sz,
filename: frame_name_list[0]
})
image_, templates_z_upper = sess.run(
[image, templates_z],
feed_dict={
siam.pos_x_ph: pos_x,
siam.pos_y_ph: pos_y - target_h / 2,
siam.z_sz_ph: z_sz,
filename: frame_name_list[0]
})
image_, templates_z_lower = sess.run(
[image, templates_z],
feed_dict={
siam.pos_x_ph: pos_x,
siam.pos_y_ph: pos_y + target_h / 2,
siam.z_sz_ph: z_sz,
filename: frame_name_list[0]
})
new_templates_z_ = templates_z_
new_templates_z_upper = templates_z_upper
new_templates_z_lower = templates_z_lower
t_start = time.time()
sco_final = np.zeros((3, 257, 257))
# Get an image from the queue
for i in range(1, num_frames):
for j in range(1, 4):
scaled_exemplar = z_sz * scale_factors
scaled_search_area = x_sz * scale_factors
scaled_target_w = target_w * scale_factors
scaled_target_h = target_h * scale_factors
image_, scores_ = sess.run(
[image, scores],
feed_dict={
siam.pos_x_ph:
pos_x,
#siam.pos_y_ph: pos_y,
siam.pos_y_ph:
pos_y - target_h / 2 if j == 1 else
(pos_y + target_h / 2 if j == 2 else pos_y),
siam.x_sz0_ph:
scaled_search_area[0],
siam.x_sz1_ph:
scaled_search_area[1],
siam.x_sz2_ph:
scaled_search_area[2],
templates_z:
np.squeeze(templates_z_upper) if j == 1 else
(np.squeeze(templates_z_lower)
if j == 2 else np.squeeze(templates_z_)),
filename:
frame_name_list[i],
},
**run_opts)
if j == 1:
templates_zupper = np.squeeze(templates_z_upper)
templates_zupper = tf.convert_to_tensor(
templates_zupper, np.float32)
elif j == 2:
templates_zlower = np.squeeze(templates_z_lower)
templates_zlower = tf.convert_to_tensor(
templates_zlower, np.float32)
else:
templates_zmain = np.squeeze(templates_z_)
templates_zmain = tf.convert_to_tensor(
templates_zmain, np.float32)
scores_ = np.squeeze(scores_)
# penalize change of scale
scores_[0, :, :] = hp.scale_penalty * scores_[0, :, :]
scores_[2, :, :] = hp.scale_penalty * scores_[2, :, :]
# find scale with highest peak (after penalty)
new_scale_id = np.argmax(np.amax(scores_, axis=(1, 2)))
# update scaled sizes
x_sz = (
1 - hp.scale_lr
) * x_sz + hp.scale_lr * scaled_search_area[new_scale_id]
target_w = (
1 - hp.scale_lr
) * target_w + hp.scale_lr * scaled_target_w[new_scale_id]
target_h = (
1 - hp.scale_lr
) * target_h + hp.scale_lr * scaled_target_h[new_scale_id]
# select response with new_scale_id
score_ = scores_[new_scale_id, :, :]
score_ = score_ - np.min(score_)
score_ = score_ / np.sum(score_)
# apply displacement penalty
#score_ = (1-hp.window_influence)*score_ + hp.window_influence*penalty
min1 = score_.min()
max1 = score_.max()
#score_max = skimage.measure.block_reduce(score_, (5,5), np.max)
#score_max=ndimage.distance_transform_edt(score_)
w = [0.1, 0, 0.1, 0]
score_ = (score_ / score_.max()) * 255
score_max = dt2d(score_, w, 4)
score_max = (((score_max - min1) *
(score_max.max() - score_max.min())) /
(max1 - min1)) + score_max.min()
#score_max_norm = Image.fromarray(score_max)
new_width = 257
new_height = 257
#sco = score_max_norm.resize((new_width,new_height),Image.ANTIALIAS)
#sco = resize(int(score_max), (257, 257))
sco = score_max
sco_final[j - 1, :, :] = sco
#####################################################################################
sco_f = sco_final[0, :, :] + sco_final[1, :, :] + sco_final[
2, :, :]
#sco_f = sco_final[0,:,:]
pos_x, pos_y = _update_target_position(pos_x, pos_y, sco_f,
final_score_sz,
design.tot_stride,
design.search_sz,
hp.response_up, x_sz)
pos_x_upper, pos_y_upper = _update_target_position(
pos_x, pos_y, sco_final[0, :, :], final_score_sz,
design.tot_stride, design.search_sz, hp.response_up, x_sz)
pos_x_lower, pos_y_lower = _update_target_position(
pos_x, pos_y, sco_final[1, :, :], final_score_sz,
design.tot_stride, design.search_sz, hp.response_up, x_sz)
# convert <cx,cy,w,h> to <x,y,w,h> and save output
bboxes[
i, :] = pos_x - target_w / 2, pos_y - target_h / 2, target_w, target_h
bboxesupper[
i, :] = pos_x_upper - target_w / 2, pos_y_upper - target_h / 2, target_w, target_h / 2
bboxeslower[
i, :] = pos_x_lower - target_w / 2, pos_y_lower, target_w, target_h / 2
# update the target representation with a rolling average
if hp.z_lr > 0:
new_templates_z_ = sess.run(
[templates_zmain],
feed_dict={
siam.pos_x_ph: pos_x,
siam.pos_y_ph: pos_y,
siam.z_sz_ph: z_sz,
image: image_
})
new_templates_z_upper = sess.run(
[templates_zupper],
feed_dict={
siam.pos_x_ph: pos_x_upper,
siam.pos_y_ph: pos_y_upper,
siam.z_sz_ph: z_sz,
image: image_
})
new_templates_z_lower = sess.run(
[templates_zlower],
feed_dict={
siam.pos_x_ph: pos_x_lower,
siam.pos_y_ph: pos_y_lower,
siam.z_sz_ph: z_sz,
image: image_
})
templates_z_ = (1 - hp.z_lr) * np.asarray(
templates_z_) + hp.z_lr * np.asarray(new_templates_z_)
templates_z_upper = (1 - hp.z_lr) * np.asarray(
templates_z_upper) + hp.z_lr * np.asarray(
new_templates_z_upper)
templates_z_lower = (1 - hp.z_lr) * np.asarray(
templates_z_lower) + hp.z_lr * np.asarray(
new_templates_z_lower)
# update template patch size
z_sz = (1 - hp.scale_lr
) * z_sz + hp.scale_lr * scaled_exemplar[new_scale_id]
if run.visualization:
show_frame(image_, bboxes[i, :], bboxesupper[i, :],
bboxeslower[i, :], 1)
t_elapsed = time.time() - t_start
speed = num_frames / t_elapsed
# Finish off the filename queue coordinator.
coord.request_stop()
coord.join(threads)
# from tensorflow.python.client import timeline
# trace = timeline.Timeline(step_stats=run_metadata.step_stats)
# trace_file = open('timeline-search.ctf.json', 'w')
# trace_file.write(trace.generate_chrome_trace_format())
plt.close('all')
return bboxes, speed
'max_ego_spawn_times': 200, # maximum times to spawn ego vehicle
'display_route': True, # whether to render the desired route
'pixor_size': 64, # size of the pixor labels
'pixor': False, # whether to output PIXOR observation
}
# Set gym-carla environment
env = gym.make('carla-v0', params=params)
return env
env = createEnv()
actor = load_model("agent/saved/final_carla_target_actor")
print("Loaded the moduel actor")
actor.summary()
print("Running the saved model")
# evaluate loaded model on test data
for i in range(10):
done = False
prev_state = env.reset()['birdeye']
print(f"running episode {i}")
while not done:
tf_prev_state = tf.expand_dims(tf.convert_to_tensor(prev_state), 0)
action = actor.predict(tf_prev_state)
prev_state, reward, done, info = env.step(action[0])
prev_state = prev_state['birdeye']
# env.render()
env.close()
def test_private(self):
prot = tfe.protocol.SecureNN()
bit_dtype = prot.prime_factory
val_dtype = prot.tensor_factory
x = np.array([
21,
21,
21,
21,
21,
21,
21,
21
], dtype=np.int32).reshape(2, 2, 2)
r = np.array([
36,
20,
21,
22,
36,
20,
21,
22
], dtype=np.int32).reshape(2, 2, 2)
beta = np.array([
0,
0,
0,
0,
1,
1,
1,
1
], dtype=np.int32).reshape(2, 2, 2)
expected = np.bitwise_xor(x > r, beta.astype(bool)).astype(np.int32)
res = _private_compare(
prot,
x_bits=PondPrivateTensor(
prot,
*prot._share(val_dtype.tensor(tf.convert_to_tensor(x, dtype=val_dtype.native_type)).to_bits(bit_dtype)),
False),
r=PondPublicTensor(
prot,
val_dtype.tensor(tf.convert_to_tensor(r, dtype=val_dtype.native_type)),
val_dtype.tensor(tf.convert_to_tensor(r, dtype=val_dtype.native_type)),
False),
beta=PondPublicTensor(
prot,
bit_dtype.tensor(tf.convert_to_tensor(beta, dtype=bit_dtype.native_type)),
bit_dtype.tensor(tf.convert_to_tensor(beta, dtype=bit_dtype.native_type)),
False)
)
with tfe.Session() as sess:
actual = sess.run(res.reveal().value_on_0.to_native())
np.testing.assert_array_equal(actual, expected)
def main():
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
if a.mode == "export":
# export the generator to a meta graph that can be imported later for standalone generation
if a.lab_colorization:
raise Exception("export not supported for lab_colorization")
input = tf.placeholder(tf.string, shape=[1])
input_data = tf.decode_base64(input[0])
input_image = tf.image.decode_png(input_data)
# remove alpha channel if present
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 4), lambda: input_image[:,:,:3], lambda: input_image)
# convert grayscale to RGB
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1), lambda: tf.image.grayscale_to_rgb(input_image), lambda: input_image)
input_image = tf.image.convert_image_dtype(input_image, dtype=tf.float32)
input_image.set_shape([CROP_SIZE, CROP_SIZE, 3])
batch_input = tf.expand_dims(input_image, axis=0)
with tf.variable_scope("generator"):
batch_output = deprocess(create_generator(preprocess(batch_input), 3))
output_image = tf.image.convert_image_dtype(batch_output, dtype=tf.uint8)[0]
if a.output_filetype == "png":
output_data = tf.image.encode_png(output_image)
elif a.output_filetype == "jpeg":
output_data = tf.image.encode_jpeg(output_image, quality=80)
else:
raise Exception("invalid filetype")
output = tf.convert_to_tensor([tf.encode_base64(output_data)])
key = tf.placeholder(tf.string, shape=[1])
inputs = {
"key": key.name,
"input": input.name
}
tf.add_to_collection("inputs", json.dumps(inputs))
outputs = {
"key": tf.identity(key).name,
"output": output.name,
}
tf.add_to_collection("outputs", json.dumps(outputs))
init_op = tf.global_variables_initializer()
restore_saver = tf.train.Saver()
export_saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
restore_saver.restore(sess, checkpoint)
print("exporting model")
export_saver.export_meta_graph(filename=os.path.join(a.output_dir, "export.meta"))
export_saver.save(sess, os.path.join(a.output_dir, "export"), write_meta_graph=False)
return
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputs, examples.targets)
# undo colorization splitting on images that we use for display/output
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(model.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths": examples.paths,
"inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"),
"targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"),
"outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"),
}
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
with tf.name_scope("predict_real_summary"):
tf.summary.image("predict_real", tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image("predict_fake", tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches, options=options, run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"], step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch)
train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print("progress epoch %d step %d image/sec %0.1f remaining %dm" % (train_epoch, train_step, rate, remaining / 60))
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
if should(a.save_freq):
print("saving model")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
x = self.branch2c(x)
x = self.branch2c_bn(x)
x = x + self.shortcut_func(inputs)
x = self.relu(x)
return x
def parse_inputs(inputs):
pass
if __name__ == '__main__':
import numpy as np
tf.enable_eager_execution()
image = np.arange(25 * 4).reshape((5, 5, 4)).astype(np.float32)
image = tf.convert_to_tensor(image[None, :, :, :])
bottle = Bottleneck(1, name='res2_0')
a = bottle(image)
print(bottle.weights)
for var in bottle.weights:
print(tuple(var.shape))
print(bottle.weights)
def generator_fn():
for melody in melodies:
yield tf.convert_to_tensor([melody])
def interpolate(x,
x_data,
y_data,
left_slope=0.0,
right_slope=0.0,
validate_args=False,
dtype=None,
name=None):
"""Performs linear interpolation for supplied points.
Given a set of knots whose x- and y- coordinates are in `x_data` and `y_data`,
this function returns y-values for x-coordinates in `x` via piecewise
linear interpolation.
`x_data` must be non decreasing, but `y_data` don't need to be because we do
not require the function approximated by these knots to be monotonic.
#### Examples
```python
x = [-10, -1, 1, 3, 6, 7, 8, 15, 18, 25, 30, 35]
x_data = [-1, 2, 6, 8, 18, 30.0]
y_data = [10, -1, -5, 7, 9, 20]
result = linear_interpolation(x, x_data, y_data)
# [ 10, 10, 2.66666667, -2, -5, 1, 7, 8.4, 9, 15.41666667, 20, 20]
```
Args:
x: x-coordinates for which we need to get interpolation. A N-D `Tensor` of
real dtype. First N-1 dimensions represent batching dimensions.
x_data: x coordinates. A N-D `Tensor` of real dtype. Should be sorted
in non decreasing order. First N-1 dimensions represent batching
dimensions.
y_data: y coordinates. A N-D `Tensor` of real dtype. Should have the
compatible shape as `x_data`. First N-1 dimensions represent batching
dimensions.
left_slope: The slope to use for extrapolation with x-coordinate smaller
than the min `x_data`. It's a 0-D or N-D `Tensor`. If not supplied, the
default will be 0, meaning constant extrapolation, i.e. extrapolated value
will be the leftmost `y_data`.
right_slope: The slope to use for extrapolation with x-coordinate greater
than the max `x_data`. It's a 0-D or N-D `Tensor`. If not supplied, the
default will be 0, meaning constant extrapolation, i.e. extrapolated value
will be the rightmost `y_data`.
validate_args: Python `bool` that indicates whether the function performs
the check if the elements in `x_data` are non decreasing. If this value is
set to `False` and the elements in x_data are not increasing, the result
of linear interpolation may be wrong.
dtype: Optional tf.dtype for `x`, x_data`, `y_data`, `left_slope` and
`right_slope`. If not specified, the dtype of the inputs will be used.
name: Python str. The name prefixed to the ops created by this function. If
not supplied, the default name 'linear_interpolation' is used.
Returns:
A N-D `Tensor` of real dtype corresponding to the x-values in `x`.
"""
with tf.compat.v1.name_scope(
name,
default_name='linear_interpolation',
values=[x, x_data, y_data, left_slope, right_slope]):
x = tf.convert_to_tensor(x, dtype=dtype)
x_data = tf.convert_to_tensor(x_data, dtype=dtype)
y_data = tf.convert_to_tensor(y_data, dtype=dtype)
expected_output_shape = tf.shape(x)
# Reshape input parameters to support batching.
x = tf.reshape(x, [-1, tf.shape(x)[-1]])
x_data = tf.reshape(x_data, [-1, tf.shape(x_data)[-1]])
y_data = tf.reshape(y_data, [-1, tf.shape(y_data)[-1]])
# Broadcast slope arguments to batch dimension.
broadcasted_slope_shape = [tf.shape(x)[0], 1]
left_slope = tf.broadcast_to(
tf.convert_to_tensor(left_slope, dtype=dtype), broadcasted_slope_shape)
right_slope = tf.broadcast_to(
tf.convert_to_tensor(right_slope, dtype=dtype), broadcasted_slope_shape)
def calc(args):
return _linear_interpolation_single_batch(args[0], args[1], args[2],
args[3], args[4], validate_args)
with tf.control_dependencies(
[tf.compat.v1.assert_equal(tf.shape(x_data), tf.shape(y_data))]):
# Call 1D linear interpolation function for each batch separately.
return tf.reshape(
tf.map_fn(
calc, (x, x_data, y_data, left_slope, right_slope),
dtype=x.dtype), expected_output_shape)
def batch_greedy_decode(model, enc_data, vocab, params):
# 判断输入长度
#print("\n enc_data\n",enc_data)
'''
#如果送进来的数据是for batch in dataset产生的，则需要
batch_data = enc_data[0]["enc_input"]#shape=(batch_size,实际输入序列长度)
batch_size = enc_data[0]["enc_input"].shape[0]
'''
batch_data = enc_data["enc_input"]#shape=(batch_size,实际输入序列长度)
batch_size = enc_data["enc_input"].shape[0]
# 开辟结果存储list
predicts = [''] * batch_size#也是一批一起运算，大小为batch_size
#print("batch_data,batch_size",batch_data,batch_size)
# inputs = batch_data # shape=(batch_size,实际序列长度)
inputs = tf.convert_to_tensor(batch_data)#
# hidden = [tf.zeros((batch_size, params['enc_units']))]
# enc_output, enc_hidden = model.encoder(inputs, hidden)
enc_output, enc_hidden = model.call_encoder(inputs)#enc_output.shape=(batch_size,实际长度,enc_hidden_size),enc_hidden.shape=dec_hidden.shape=(batch_size,hidden_size)
dec_hidden = enc_hidden
#print("inputs,enc_output,enc_hidden,dec_hidden",inputs.get_shape(),enc_output.get_shape(),enc_hidden.get_shape(),dec_hidden.get_shape())
# dec_input = tf.expand_dims([vocab.word_to_id(vocab.START_DECODING)] * batch_size, 1)
dec_input = tf.constant([2] * batch_size)
dec_input = tf.expand_dims(dec_input, axis=1)#shape=(batch_size,1)
context_vector, _ = model.attention(dec_hidden, enc_output)#shape=(batch_size,enc_hidden_size)
# print("dec_inp,contexy_vector",dec_input.get_shape(),context_vector.get_shape())
#核心套路：一批数据先进来统一做一个encoder,在统一做一个attention得到一个初始的context_vector，再对每一批数据的一个个字符进行decoder、attention,得到那一批数据在同一位置的decoder输出值
#print("greedy_search.shape:",dec_input.get_shape(),dec_hidden.get_shape(),enc_output.get_shape(),context_vector.get_shape())
#greedy_search.shape: (16, 1) (16, 256) (16, 184, 256) (16, 256)
for t in range(params['max_dec_len']):
# 单步预测
#通过调用decoder得到预测的概率分布,参考训练阶段
_, pred, dec_hidden = model.decoder(dec_input,
dec_hidden,
enc_output,
context_vector)
context_vector, _ = model.attention(dec_hidden, enc_output)
#通过调用tf.argmax完成greedy search，得到predicted_ids，找到概率值最大的作为预测的输出
predicted_ids = tf.argmax(pred,axis=1).numpy()
#将每一步的预测结果都保存起来
for index, predicted_id in enumerate(predicted_ids):
#print("predicted_id=",predicted_id)#不要瞎打印，不然整个计算过程会很慢
predicts[index] += vocab.id_to_word(predicted_id) + ' '#此处注意设置字典大小最好为实际大小，不然会提示找不到对应的词
# 前一步的预测结果作为下一步的输入
dec_input = tf.expand_dims(predicted_ids, 1)
results = []
for predict in predicts:
# 去掉句子前后空格
predict = predict.strip()
print("pre",predict,type(predict))
# 句子小于max len就结束了 截断vocab.word_to_id('[STOP]')
# if '[STOP]' in predict:
# # 截断stop
# predict = predict[:predict.index('[STOP]')]
# 保存结果
results.append(predict)
print("results:",results,len(results))
return results
def generator_fn():
for key1_d, key2_d in zip(key1_data, key2_data):
yield tf.convert_to_tensor([key1_d
]), tf.convert_to_tensor([key2_d])
for vector_raw in list(zip(*vectors)):
vector = []
for element in vector_raw:
if element == "None" or element is None:
element = float(0)
print(element)
vector.append(float(element))
formated.append(vector)
return formated
supervised = make_supervised(new_frame)
structed_inputs = encode(supervised["inputs"])
structed_outputs = encode(supervised["outputs"])
train_x = tf.convert_to_tensor(np.array(structed_inputs[:2000],
dtype=np.float))
train_y = tf.convert_to_tensor(
np.array(structed_outputs[:2000], dtype=np.float))
test_x = np.array(structed_inputs[2000:], dtype=np.float)
test_y = np.array(structed_outputs[2000:], dtype=np.float)
opt = SGD(lr=0.01, momentum=0.9)
model = ks.Sequential()
model.add(ks.layers.core.Dense(units=10, activation="relu"))
model.add(ks.layers.core.Dense(units=2, activation="sigmoid"))
model.compile(loss="mse", optimizer=opt, metrics=["acc"])
fit_results = model.fit(x=train_x,
y=train_y,
validation_data=(test_x, test_y),
def train():
conf = config()
for n in (2, ):
# Models to be compared
# d = { "LSTM": 0, "FUN": 0}
d = {"FUN": 0}
# Generate a (binary problem) problem
x, y = generateBaseProblem(n)
print "GENERATED PROBLEM:"
for sequence_size in (15, ):
for model_name in d.keys():
acc = 0
lr_list = (1e-3, )
for learning_rate in lr_list:
for tentativo in range(1):
print(
"n=%d | sequence_size=%d | model_name=%s | tentativo=%d | learning_rate=%f"
% (n, sequence_size, model_name, tentativo,
learning_rate))
# reset the default graph to build an (eventually) new one
tf.reset_default_graph()
m = STATIC_RNN(conf,
learning_rate=learning_rate,
sequence_size=sequence_size,
model_name=model_name)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for e in range(conf.EPOCHS):
acc, loss = 0, 0
for k in range(conf.dataset_size //
conf.batch_size):
# generate new batch
batch_X, batch_Y = generateBatch(
x, y, sequence_size, conf.batch_size)
batch_X = np.reshape(
batch_X,
[conf.batch_size, sequence_size, 1])
batch_Y = np.reshape(
batch_Y, [conf.batch_size, 1])
_, acc_, loss_ = sess.run(
(m.train_op, m.accuracy, m.loss),
feed_dict={
m.x: batch_X,
m.y: batch_Y
})
acc, loss = acc + acc_, loss + loss_
acc = acc / (conf.dataset_size //
conf.batch_size)
loss = loss / (conf.dataset_size //
conf.batch_size)
if (e == 0) or (e % conf.plot_each
== conf.plot_each - 1):
print "Epoch =", e + 1, "\t| Loss =", loss, "\t| Accuracy (train) =", acc
if heardEnter(): break
if np.isnan(loss): break
if acc > 99.: break
# saver.save(sess, "savings/model.ckpt")
A_START, A_STOP = -5, 5
A_STEP = abs(A_STOP - A_START) / 1000.
A = np.reshape(
np.arange(start=A_START,
stop=A_STOP,
step=A_STEP,
dtype=np.float32), [-1, 1])
A = np.tile(A, [1, 20])
A_tensor = tf.convert_to_tensor(A)
with tf.variable_scope(
"rnn/fun_rnn_cell/FUNCell/state",
reuse=True):
y_ = fun.fun_activation(
A_tensor,
k=config.k,
lambda_alphas=config.lambda_alphas)
grad = tf.gradients(y_, A_tensor)
y_, grad = sess.run((y_, grad[0]))
with tf.variable_scope(
"rnn/fun_rnn_cell/FUNCell/bn", reuse=True):
moving_means = tf.get_variable("moving_mean")
print(sess.run(moving_means))
_, ax = plt.subplots(5, 4)
for i in range(conf.hidden_size):
ax[i // 4][i % 4].plot(A[:, i], y_[:, i])
# ax[i // 4][i % 4].set_ylim([-5 ,5])
# ax[i // 4][i % 4].axes.xaxis.set_ticklabels([])
# ax[i // 4][i % 4].axes.yaxis.set_ticklabels([])
plt.show()
plt.close()
_, ax = plt.subplots(5, 4)
for i in range(conf.hidden_size):
ax[i // 4][i % 4].plot(A[:, i], grad[:, i])
ax[i // 4][i % 4].set_ylim([-3, 3])
ax[i // 4][i % 4].grid()
ax[i // 4][i % 4].set_xticks(
np.arange(-5, 5, 1))
ax[i // 4][i % 4].set_yticks(
np.arange(-3, 3., 1))
# ax[i // 4][i % 4].axes.xaxis.set_ticklabels([])
# ax[i // 4][i % 4].axes.yaxis.set_ticklabels([])
plt.show()
