def test_docstring_example(self):
# Produce the first 1000 members of the Halton sequence in 3 dimensions.
num_results = 1000
dim = 3
with self.test_session():
sample = tfp.mcmc.sample_halton_sequence(
dim, num_results=num_results, randomized=False)
# Evaluate the integral of x_1 * x_2^2 * x_3^3 over the three dimensional
# hypercube.
powers = tf.range(1., limit=dim + 1)
integral = tf.reduce_mean(
tf.reduce_prod(sample ** powers, axis=-1))
true_value = 1. / tf.reduce_prod(powers + 1.)
# Produces a relative absolute error of 1.7%.
self.assertAllClose(integral.eval(), true_value.eval(), rtol=0.02)
# Now skip the first 1000 samples and recompute the integral with the next
# thousand samples. The sequence_indices argument can be used to do this.
sequence_indices = tf.range(start=1000, limit=1000 + num_results,
dtype=tf.int32)
sample_leaped = tfp.mcmc.sample_halton_sequence(
dim, sequence_indices=sequence_indices, randomized=False)
integral_leaped = tf.reduce_mean(
tf.reduce_prod(sample_leaped ** powers, axis=-1))
self.assertAllClose(integral_leaped.eval(), true_value.eval(), rtol=0.05)
def testArgRenames(self):
with self.test_session():
a = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
b = [[True, False, False], [False, True, True]]
dim0 = [1]
dim1 = [1]
self.assertAllEqual(tf.reduce_any(b, reduction_indices=dim0).eval(), [True, True])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=[0]).eval(), [False, False, False])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=dim1).eval(), [False, False])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[1]).eval(), [6.0, 15.0])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_sum(a, [0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[1]).eval(), [6.0, 120.0])
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_prod(a, [0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[1]).eval(), [2.0, 5.0])
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_mean(a, [0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[1]).eval(), [1.0, 4.0])
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_min(a, [0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[1]).eval(), [3.0, 6.0])
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[0, 1]).eval(), 6.0)
self.assertAllEqual(tf.reduce_max(a, [0, 1]).eval(), 6.0)
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[1]).eval(), [3.40760589, 6.40760612])
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[0, 1]).eval(), 6.45619344711)
self.assertAllClose(tf.reduce_logsumexp(a, [0, 1]).eval(), 6.45619344711)
self.assertAllEqual(tf.expand_dims([[1, 2], [3, 4]], dim=1).eval(), [[[1, 2]], [[3, 4]]])
def disjunction_of_literals(literals, label="no_label"):
list_of_literal_tensors = [lit.tensor for lit in literals]
literals_tensor = tf.concat(1,list_of_literal_tensors)
if default_tnorm == "product":
result = 1.0-tf.reduce_prod(1.0-literals_tensor, 1, keep_dims=True)
if default_tnorm == "yager2":
result = tf.minimum(1.0, tf.sqrt(tf.reduce_sum(tf.square(literals_tensor), 1, keep_dims=True)))
if default_tnorm == "luk":
print "data aggregator is lukas"
result = tf.minimum(1.0, tf.reduce_sum(literals_tensor, 1, keep_dims=True))
PR(result)
if default_tnorm == "goedel":
result = tf.reduce_max(literals_tensor, 1, keep_dims=True, name=label)
if default_aggregator == "product":
return tf.reduce_prod(result, keep_dims=True)
if default_aggregator == "mean":
print "data aggregator is mean"
return tf.reduce_mean(result, keep_dims=True, name=label)
if default_aggregator == "gmean":
return tf.exp(tf.mul(tf.reduce_sum(tf.log(result), keep_dims=True),
tf.inv(tf.to_float(tf.size(result)))), name=label)
if default_aggregator == "hmean":
print "data aggregator is hmean"
return tf.div(tf.to_float(tf.size(result)), tf.reduce_sum(tf.inv(result), keep_dims=True))
if default_aggregator == "min":
print "data aggregator is min"
return tf.reduce_min(result, keep_dims=True, name=label)
if default_aggregator == "qmean":
print "data aggregator is qmean"
return tf.sqrt(tf.reduce_mean(tf.square(result), keep_dims=True), name=label)
if default_aggregator == "cmean":
print "data aggregator is cmean"
return tf.pow(tf.reduce_mean(tf.pow(result, 3), keep_dims=True), tf.inv(tf.to_float(3)), name=label)
def U_t_variance(timestep_outputs_matrix, total_timesteps, gamma = 5): with tf.op_scope(timestep_outputs_matrix + total_timesteps + gamma, "U_t_variance"): G_i_matrix = G_i_piecewise_variance(timestep_outputs_matrix, total_timesteps) tf.mul(timestep_outputs_matrix, ) tf.reduce_prod(timestep_outputs_matrix_with_g)
def tf_bivariate_normal(y, mu, sigma, rho, n_mixtures, batch_size):
mu = tf.verify_tensor_all_finite(mu, "Mu not finite!")
y = tf.verify_tensor_all_finite(y, "Y not finite!")
delta = tf.sub(tf.tile(tf.expand_dims(y, 1), [1, n_mixtures, 1]), mu)
delta = tf.verify_tensor_all_finite(delta, "Delta not finite!")
sigma = tf.verify_tensor_all_finite(sigma, "Sigma not finite!")
s = tf.reduce_prod(sigma, 2)
s = tf.verify_tensor_all_finite(s, "S not finite!")
# -1 <= rho <= 1
z = tf.reduce_sum(tf.square(tf.div(delta, sigma + epsilon) + epsilon), 2) - \
2 * tf.div(tf.mul(rho, tf.reduce_prod(delta, 2)), s + epsilon)
z = tf.verify_tensor_all_finite(z, "Z not finite!")
# 0 < negRho <= 1
rho = tf.verify_tensor_all_finite(rho, "rho in bivariate normal not finite!")
negRho = tf.clip_by_value(1 - tf.square(rho), epsilon, 1.0)
negRho = tf.verify_tensor_all_finite(negRho, "negRho not finite!")
# Note that if negRho goes near zero, or z goes really large, this explodes.
negRho = tf.verify_tensor_all_finite(negRho, "negRho in bivariate normal not finite!")
result = tf.clip_by_value(tf.exp(tf.div(-z, 2 * negRho)), 1.0e-8, 1.0e8)
result = tf.verify_tensor_all_finite(result, "Result in bivariate normal not finite!")
denom = 2 * np.pi * tf.mul(s, tf.sqrt(negRho))
denom = tf.verify_tensor_all_finite(denom, "Denom in bivariate normal not finite!")
result = tf.clip_by_value(tf.div(result, denom + epsilon), epsilon, 1.0)
result = tf.verify_tensor_all_finite(result, "Result2 in bivariate normal not finite!")
return result, delta
def f_iou_box(top_left_a, bot_right_a, top_left_b, bot_right_b):
"""Computes IoU of boxes.
Args:
top_left_a: [B, T, 2] or [B, 2]
bot_right_a: [B, T, 2] or [B, 2]
top_left_b: [B, T, 2] or [B, 2]
bot_right_b: [B, T, 2] or [B, 2]
Returns:
iou: [B, T]
"""
inter_area = f_inter_box(top_left_a, bot_right_a, top_left_b, bot_right_b)
inter_area = tf.maximum(inter_area, 1e-6)
ndims = tf.shape(tf.shape(top_left_a))
# area_a = tf.reduce_prod(bot_right_a - top_left_a, ndims - 1)
# area_b = tf.reduce_prod(bot_right_b - top_left_b, ndims - 1)
check_a = tf.reduce_prod(tf.to_float(top_left_a < bot_right_a), ndims - 1)
area_a = check_a * tf.reduce_prod(bot_right_a - top_left_a, ndims - 1)
check_b = tf.reduce_prod(tf.to_float(top_left_b < bot_right_b), ndims - 1)
area_b = check_b * tf.reduce_prod(bot_right_b - top_left_b, ndims - 1)
union_area = (area_a + area_b - inter_area + 1e-5)
union_area = tf.maximum(union_area, 1e-5)
iou = inter_area / union_area
iou = tf.maximum(iou, 1e-5)
iou = tf.minimum(iou, 1.0)
return iou
def calculate_reshape(original_shape, new_shape, validate=False, name=None):
"""Calculates the reshaped dimensions (replacing up to one -1 in reshape)."""
batch_shape_static = tensor_util.constant_value_as_shape(new_shape)
if batch_shape_static.is_fully_defined():
return np.int32(batch_shape_static.as_list()), batch_shape_static, []
with tf.name_scope(name, "calculate_reshape", [original_shape, new_shape]):
original_size = tf.reduce_prod(original_shape)
implicit_dim = tf.equal(new_shape, -1)
size_implicit_dim = (
original_size // tf.maximum(1, -tf.reduce_prod(new_shape)))
new_ndims = tf.shape(new_shape)
expanded_new_shape = tf.where( # Assumes exactly one `-1`.
implicit_dim, tf.fill(new_ndims, size_implicit_dim), new_shape)
validations = [] if not validate else [
tf.assert_rank(
original_shape, 1, message="Original shape must be a vector."),
tf.assert_rank(new_shape, 1, message="New shape must be a vector."),
tf.assert_less_equal(
tf.count_nonzero(implicit_dim, dtype=tf.int32),
1,
message="At most one dimension can be unknown."),
tf.assert_positive(
expanded_new_shape, message="Shape elements must be >=-1."),
tf.assert_equal(
tf.reduce_prod(expanded_new_shape),
original_size,
message="Shape sizes do not match."),
]
return expanded_new_shape, batch_shape_static, validations
def build_psi_stats_rbf_plus_linear(Z, kern, mu, S):
# TODO: make sure the acvite dimensions are overlapping completely
# use only active dimensions
mu, S = kern._slice(mu, S) # only use the active dimensions.
Z, _ = kern._slice(Z, None)
psi0_lin, psi1_lin, psi2_lin = build_psi_stats_linear(Z, kern.linear, mu, S)
psi0_rbf, psi1_rbf, psi2_rbf = build_psi_stats_rbf(Z, kern.rbf, mu, S)
psi0, psi1, psi2 = psi0_lin + psi0_rbf, psi1_lin + psi1_rbf, psi2_lin + psi2_rbf
# extra terms for the 'interaction' of linear and rbf
l2 = tf.square(kern.rbf.lengthscales)
A = tf.expand_dims(1./S + 1./l2, 1) # N x 1 x Q
m = (tf.expand_dims(mu/S, 1) + tf.expand_dims(Z/l2, 0)) / A # N x M x Q
mTAZ = tf.reduce_sum(tf.expand_dims(m * kern.linear.variance, 1) *
tf.expand_dims(tf.expand_dims(Z, 0), 0), 3) # N x M x M
Z2 = tf.reduce_sum(tf.square(Z) / l2, 1) # M,
mu2 = tf.reduce_sum(tf.square(mu) / S, 1) # N
mAm = tf.reduce_sum(tf.square(m) * A, 2) # N x M
exp_term = tf.exp(-(tf.reshape(Z2, (1, -1)) + tf.reshape(mu2, (-1, 1))-mAm) / 2.) # N x M
psi2_extra = tf.reduce_sum(kern.rbf.variance *
tf.expand_dims(exp_term, 2) *
tf.expand_dims(tf.expand_dims(tf.reduce_prod(S, 1), 1), 2) *
tf.expand_dims(tf.reduce_prod(A, 2), 1) *
mTAZ, 0)
psi2 = psi2 + psi2_extra + tf.transpose(psi2_extra)
return psi0, psi1, psi2
def testGradient(self):
s = [2, 3, 4, 2]
# NOTE(kearnes): divide by 20 so product is a reasonable size
x = np.arange(1.0, 49.0).reshape(s).astype(np.float32) / 20.
with self.test_session():
t = tf.convert_to_tensor(x)
su = tf.reduce_prod(t, [])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [2, 3, 4, 2], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [1, 2])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [2, 2], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [0, 1, 2, 3])
jacob_t, jacob_n = gradient_checker.ComputeGradient(
t, s, su, [1], x_init_value=x, delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
# NOTE(kearnes): the current gradient calculation gives NaNs for 0 inputs
x = np.arange(0.0, 48.0).reshape(s).astype(np.float32) / 20.
with self.test_session():
t = tf.convert_to_tensor(x)
su = tf.reduce_prod(t, [])
jacob_t, _ = gradient_checker.ComputeGradient(
t, s, su, [2, 3, 4, 2], x_init_value=x, delta=1)
with self.assertRaisesOpError("Tensor had NaN values"):
tf.check_numerics(jacob_t, message="_ProdGrad NaN test").op.run()
def _compareGradient(self, x):
with self.test_session():
t = tf.convert_to_tensor(x)
su = tf.reduce_prod(t, [])
jacob_t, jacob_n = tf.test.compute_gradient(t,
x.shape,
su,
[2, 3, 4, 2],
x_init_value=x,
delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [1, 2])
jacob_t, jacob_n = tf.test.compute_gradient(t,
x.shape,
su,
[2, 2],
x_init_value=x,
delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
su = tf.reduce_prod(t, [0, 1, 2, 3])
jacob_t, jacob_n = tf.test.compute_gradient(t,
x.shape,
su,
[1],
x_init_value=x,
delta=1)
self.assertAllClose(jacob_t, jacob_n, rtol=1e-3, atol=1e-3)
def logpdf(self, x, mean=None, cov=1):
"""Log of the probability density function.
Parameters
----------
x : tf.Tensor
A 1-D or 2-D tensor.
mean : tf.Tensor, optional
A 1-D tensor. Defaults to zero mean.
cov : tf.Tensor, optional
A 1-D or 2-D tensor. Defaults to identity matrix.
Returns
-------
tf.Tensor
A tensor of one dimension less than the input.
"""
x = tf.cast(x, dtype=tf.float32)
x_shape = get_dims(x)
if len(x_shape) == 1:
d = x_shape[0]
else:
d = x_shape[1]
if mean is None:
r = x
else:
mean = tf.cast(mean, dtype=tf.float32)
r = x - mean
if cov is 1:
L_inv = tf.diag(tf.ones([d]))
det_cov = tf.constant(1.0)
else:
cov = tf.cast(cov, dtype=tf.float32)
if len(cov.get_shape()) == 1: # vector
L_inv = tf.diag(1.0 / tf.sqrt(cov))
det_cov = tf.reduce_prod(cov)
else: # matrix
L = tf.cholesky(cov)
L_inv = tf.matrix_inverse(L)
det_cov = tf.pow(tf.reduce_prod(tf.diag_part(L)), 2)
lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov)
if len(x_shape) == 1: # vector
r = tf.reshape(r, shape=(d, 1))
inner = tf.matmul(L_inv, r)
lps -= 0.5 * tf.matmul(inner, inner, transpose_a=True)
return tf.squeeze(lps)
else: # matrix
# TODO vectorize further
out = []
for r_vec in tf.unpack(r):
r_vec = tf.reshape(r_vec, shape=(d, 1))
inner = tf.matmul(L_inv, r_vec)
out += [tf.squeeze(lps -
0.5 * tf.matmul(inner, inner, transpose_a=True))]
return tf.pack(out)
def _variance(self):
with tf.control_dependencies(self._runtime_assertions):
probs = self._marginal_hidden_probs()
# probs :: num_steps batch_shape num_states
means = self._observation_distribution.mean()
# means :: observation_batch_shape[:-1] num_states
# observation_event_shape
means_shape = tf.concat(
[self.batch_shape_tensor(),
[self._num_states],
self._observation_distribution.event_shape_tensor()],
axis=0)
means = tf.broadcast_to(means, means_shape)
# means :: batch_shape num_states observation_event_shape
observation_event_shape = (
self._observation_distribution.event_shape_tensor())
batch_size = tf.reduce_prod(self.batch_shape_tensor())
flat_probs_shape = [self._num_steps, batch_size, self._num_states]
flat_means_shape = [
batch_size,
1,
self._num_states,
tf.reduce_prod(observation_event_shape)]
flat_probs = tf.reshape(probs, flat_probs_shape)
# flat_probs :: num_steps batch_size num_states
flat_means = tf.reshape(means, flat_means_shape)
# flat_means :: batch_size 1 num_states observation_event_size
flat_mean = tf.einsum("ijk,jmkl->jiml", flat_probs, flat_means)
# flat_mean :: batch_size num_steps 1 observation_event_size
variances = self._observation_distribution.variance()
variances = tf.broadcast_to(variances, means_shape)
# variances :: batch_shape num_states observation_event_shape
flat_variances = tf.reshape(variances, flat_means_shape)
# flat_variances :: batch_size 1 num_states observation_event_size
# For a mixture of n distributions with mixture probabilities
# p[i], and where the individual distributions have means and
# variances given by mean[i] and var[i], the variance of
# the mixture is given by:
#
# var = sum i=1..n p[i] * ((mean[i] - mean)**2 + var[i]**2)
flat_variance = tf.einsum("ijk,jikl->jil",
flat_probs,
(flat_means - flat_mean)**2 + flat_variances)
# flat_variance :: batch_size num_steps observation_event_size
unflat_mean_shape = tf.concat(
[self.batch_shape_tensor(),
[self._num_steps],
observation_event_shape],
axis=0)
# returns :: batch_shape num_steps observation_event_shape
return tf.reshape(flat_variance, unflat_mean_shape)
def reduce_logmeanexp(input_tensor, axis=None, keep_dims=False):
logsumexp = tf.reduce_logsumexp(input_tensor, axis, keep_dims)
input_tensor = tf.convert_to_tensor(input_tensor)
n = input_tensor.shape.as_list()
if axis is None:
n = tf.cast(tf.reduce_prod(n), logsumexp.dtype)
else:
n = tf.cast(tf.reduce_prod(n[axis]), logsumexp.dtype)
return -tf.log(n) + logsumexp
def get_prediction(self,items,v_arr,M):
""" items contains either a list of item or a single item
for instance items = [0,1,2] or items = 0
when predicting multiple value for the same user, it is more efficient
to use this function with a list of some values in items instead of
calling this function multiple times since it prevent from computing
the same V for each rating prediction
"""
V = createV(v_arr,M,self.m, self.K)
V = tf.sparse_tensor_to_dense(V)
den = 0
# There are two terms in the exponential that are common to each k
Exp_c = self.h_bias
for l in range(self.K):
Exp_c = tf.add(Exp_c,tf.matmul(tf.reshape(V[:,l],(1,self.m)),self.weights[l,:,:]))
if (type(items)==list) | (type(items)==np.ndarray) | (type(items)==range):
R = []
if (type(items)!=range):
items = list(map(int,items)) # it must be int and not np.int32
for item in items:
Gamma = tf.exp(tf.mul(V[item,:],self.v_bias[item,:]))
R_tmp = 0
for k in range(self.K):
tmp = tf.add(tf.exp(-Exp_c),tf.exp(V[item,k]*self.weights[k,item,:]))
tmp = tf.reduce_prod(tmp)
tmp = tf.reduce_prod(tmp)
tmp = Gamma[k]*tmp
den = den + tmp
R_tmp = R_tmp + (k+1)*tmp
R_tmp = R_tmp/den
R.extend([R_tmp])
elif type(items)==int:
R = 0
Gamma = tf.exp(tf.mul(V[items,:],self.v_bias[items,:]))
for k in range(self.K):
tmp = tf.add(tf.exp(-Exp_c),tf.exp(V[items,k]*self.weights[k,items,:]))
tmp = tf.reduce_prod(tmp)
tmp = tf.reduce_prod(tmp)
tmp = Gamma[k]*tmp
den = den + tmp
R = R + (k+1)*tmp
R = R/den
else:
print('type error')
return R
def variable_summaries(var, name, collection_key):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization).
Args:
- var: Tensor for variable from which we want to log.
- name: Variable name.
- collection_key: Collection to save the summary to, can be any key of
`VAR_LOG_LEVELS`.
"""
if collection_key not in VAR_LOG_LEVELS.keys():
raise ValueError('"{}" not in `VAR_LOG_LEVELS`'.format(collection_key))
collections = VAR_LOG_LEVELS[collection_key]
with tf.name_scope(name):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean, collections)
num_params = tf.reduce_prod(tf.shape(var))
tf.summary.scalar('num_params', num_params, collections)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev, collections)
tf.summary.scalar('max', tf.reduce_max(var), collections)
tf.summary.scalar('min', tf.reduce_min(var), collections)
tf.summary.histogram('histogram', var, collections)
tf.summary.scalar('sparsity', tf.nn.zero_fraction(var), collections)
def _flatten_trace_edges(self, edges: List[Edge],
new_edge_name: Optional[Text]) -> Edge:
"""Flatten trace edges into single edge.
Args:
edges: List of trace edges to flatten
new_edge_name: Optional name of the new edge created.
Returns:
new_edge: The new edge that represents the flattening of the given edges.
"""
node = edges[0].node1 # We are in the trace case, so this is the only node.
# Flatten all of the edge's axes into a a single list.
perm_back = [min(e.axis1, e.axis2) for e in edges]
perm_back += [max(e.axis1, e.axis2) for e in edges]
perm_front = set(range(len(node.edges))) - set(perm_back)
perm_front = sorted(perm_front)
perm = perm_front + perm_back
new_dim = tf.reduce_prod([tf.shape(node.tensor)[e.axis1] for e in edges])
node.reorder_axes(perm)
unaffected_shape = tf.shape(node.tensor)[:len(perm_front)]
new_shape = tf.concat([unaffected_shape, [new_dim, new_dim]], axis=-1)
node.tensor = tf.reshape(node.tensor, new_shape)
edge1 = Edge("TraceFront", node, len(perm_front))
edge2 = Edge("TraceBack", node, len(perm_front) + 1)
node.edges = node.edges[:len(perm_front)] + [edge1, edge2]
new_edge = self.connect(edge1, edge2, new_edge_name)
node.axis_names = None
return new_edge
def embedding_lookup(params, ids, name="embedding_lookup"):
"""Provides a N dimensional version of tf.embedding_lookup.
Ids are flattened to a 1d tensor before being passed to embedding_lookup
then, they are unflattend to match the original ids shape plus an extra
leading dimension of the size of the embeddings.
Args:
params: List of tensors of size D0 x D1 x ... x Dn-2 x Dn-1.
ids: N-dimensional tensor of B0 x B1 x .. x Bn-2 x Bn-1.
Must contain indexes into params.
name: Optional name for the op.
Returns:
A tensor of size B0 x B1 x .. x Bn-2 x Bn-1 x D1 x ... x Dn-2 x Dn-1 containing the values from
the params tensor(s) for indecies in ids.
Raises:
ValueError: if some parameters are invalid.
"""
with tf.op_scope([params, ids], name, "embedding_lookup"):
params = tf.convert_to_tensor(params)
ids = tf.convert_to_tensor(ids)
shape = tf.shape(ids)
ids_flat = tf.reshape(ids, tf.reduce_prod(shape, keep_dims=True))
embeds_flat = tf.nn.embedding_lookup(params, ids_flat, name)
embed_shape = tf.concat(0, [shape, [-1]])
embeds = tf.reshape(embeds_flat, embed_shape)
embeds.set_shape(ids.get_shape().concatenate(params.get_shape()[1:]))
return embeds
def ut(u, f, wr, ww):
"""
returns the updated usage vector given the previous one along with
free gates and previous read and write weightings
"""
psi_t = tf.reduce_prod(1 - f * wr, 1)
return (u + ww - u * ww) * psi_t
def multilinear(emb, tuples, l2=0):
"""
Compute the dot product of real vectors at selected embeddings
Note that this model is called Cannonical Parafac (CP), and corresponds to the "distmult" model in some scientific
publications on relational database factorization.
:param emb: embedding matrix of size [n_emb, rank] containing float numbers
:param tuples: tuple matrix of size [n_t, arity] containing integers
:param l2: optional l2 regularization strength that is added to the score. If it is different from 0, the function
returns a pair (pred, l2norm) where pred is the sample prediction, but l2norm is the l2 norm of the selected
embeddings
:return: the multilinear dot product between selected embeddings S[i] = sum_j prod_k E[I[i,k],j]
>>> emb = [[1., 1, 0, 3], [0, 1, 0, 1], [-1, 1, 1, 5]]
>>> idx = tf.Variable([[0, 1], [1, 0], [0, 2], [2, 0], [1, 2], [2, 1]])
>>> g = multilinear(emb, idx)
>>> print(tf_eval(g))
[ 4. 4. 15. 15. 6. 6.]
"""
emb_sel = tf.gather(emb, tuples)
pred = tf.reduce_sum(tf.reduce_prod(emb_sel, 1), 1)
if l2 == 0: # unregularized prediction ==> returns only the predictions
return pred
else: # l2 regularization of the selected embeddings
reg = l2 * tf.reduce_sum(tf.square(emb_sel))
return pred, reg
def _finish_prob_for_one_fiber(self, y, x, ildj, distribution_kwargs):
"""Finish computation of prob on one element of the inverse image."""
x = self._maybe_rotate_dims(x, rotate_right=True)
prob = self.distribution.prob(x, **distribution_kwargs)
if self._is_maybe_event_override:
prob = tf.reduce_prod(prob, self._reduce_event_indices)
return tf.exp(tf.cast(ildj, prob.dtype)) * prob
def gauss_kl(q_mu, q_sqrt, K):
"""
Compute the KL divergence from
q(x) = N(q_mu, q_sqrt^2)
to
p(x) = N(0, K)
We assume multiple independent distributions, given by the columns of
q_mu and the last dimension of q_sqrt.
q_mu is a matrix, each column contains a mean.
q_sqrt is a 3D tensor, each matrix within is a lower triangular square-root
matrix of the covariance of q.
K is a positive definite matrix: the covariance of p.
"""
L = tf.cholesky(K)
alpha = tf.matrix_triangular_solve(L, q_mu, lower=True)
KL = 0.5 * tf.reduce_sum(tf.square(alpha)) # Mahalanobis term.
num_latent = tf.cast(tf.shape(q_sqrt)[2], float_type)
KL += num_latent * 0.5 * tf.reduce_sum(tf.log(tf.square(tf.diag_part(L)))) # Prior log-det term.
KL += -0.5 * tf.cast(tf.reduce_prod(tf.shape(q_sqrt)[1:]), float_type) # constant term
Lq = tf.matrix_band_part(tf.transpose(q_sqrt, (2, 0, 1)), -1, 0) # force lower triangle
KL += -0.5*tf.reduce_sum(tf.log(tf.square(tf.matrix_diag_part(Lq)))) # logdet
L_tiled = tf.tile(tf.expand_dims(L, 0), tf.pack([tf.shape(Lq)[0], 1, 1]))
LiLq = tf.matrix_triangular_solve(L_tiled, Lq, lower=True)
KL += 0.5 * tf.reduce_sum(tf.square(LiLq)) # Trace term
return KL
def __init__(self, net, scope, classes, boxes_per_cell, training=False):
_, self.cell_height, self.cell_width, _ = tf.get_default_graph().get_tensor_by_name(scope + '/conv:0').get_shape().as_list()
cells = self.cell_height * self.cell_width
with tf.name_scope('regress'):
with tf.name_scope('inputs'):
end = cells * classes
self.prob = tf.reshape(net[:, :end], [-1, cells, 1, classes], name='prob')
inputs_remaining = tf.reshape(net[:, end:], [-1, cells, boxes_per_cell, 5], name='inputs_remaining')
self.iou = tf.identity(inputs_remaining[:, :, :, 0], name='iou')
self.offset_xy = tf.identity(inputs_remaining[:, :, :, 1:3], name='offset_xy')
wh01_sqrt_base = tf.identity(inputs_remaining[:, :, :, 3:], name='wh01_sqrt_base')
wh01 = tf.square(wh01_sqrt_base, name='wh01')
wh01_sqrt = tf.abs(wh01_sqrt_base, name='wh01_sqrt')
self.coords = tf.concat([self.offset_xy, wh01_sqrt], -1, name='coords')
self.wh = tf.identity(wh01 * [self.cell_width, self.cell_height], name='wh')
_wh = self.wh / 2
self.offset_xy_min = tf.identity(self.offset_xy - _wh, name='offset_xy_min')
self.offset_xy_max = tf.identity(self.offset_xy + _wh, name='offset_xy_max')
self.areas = tf.reduce_prod(self.wh, -1, name='areas')
if not training:
with tf.name_scope('detection'):
cell_xy = calc_cell_xy(self.cell_height, self.cell_width).reshape([1, cells, 1, 2])
self.xy = tf.identity(cell_xy + self.offset_xy, name='xy')
self.xy_min = tf.identity(cell_xy + self.offset_xy_min, name='xy_min')
self.xy_max = tf.identity(cell_xy + self.offset_xy_max, name='xy_max')
self.conf = tf.identity(tf.expand_dims(self.iou, -1) * self.prob, name='conf')
self.inputs = net
self.classes = classes
self.boxes_per_cell = boxes_per_cell
def _expand_sample_shape_to_vector(self, x, name):
"""Helper to `sample` which ensures input is 1D."""
x_static_val = tf.contrib.util.constant_value(x)
if x_static_val is None:
prod = tf.reduce_prod(x)
else:
prod = np.prod(x_static_val, dtype=x.dtype.as_numpy_dtype())
ndims = x.shape.ndims # != sample_ndims
if ndims is None:
# Maybe expand_dims.
ndims = tf.rank(x)
expanded_shape = util.pick_vector(
tf.equal(ndims, 0),
np.array([1], dtype=np.int32), tf.shape(x))
x = tf.reshape(x, expanded_shape)
elif ndims == 0:
# Definitely expand_dims.
if x_static_val is not None:
x = tf.convert_to_tensor(
np.array([x_static_val], dtype=x.dtype.as_numpy_dtype()),
name=name)
else:
x = tf.reshape(x, [1])
elif ndims != 1:
raise ValueError("Input is neither scalar nor vector.")
return x, prod
def __init__(self, model, mask, prob, coords, offset_xy_min, offset_xy_max, areas):
self.model = model
with tf.name_scope('true'):
self.mask = tf.identity(mask, name='mask')
self.prob = tf.identity(prob, name='prob')
self.coords = tf.identity(coords, name='coords')
self.offset_xy_min = tf.identity(offset_xy_min, name='offset_xy_min')
self.offset_xy_max = tf.identity(offset_xy_max, name='offset_xy_max')
self.areas = tf.identity(areas, name='areas')
with tf.name_scope('iou') as name:
_offset_xy_min = tf.maximum(model.offset_xy_min, self.offset_xy_min, name='_offset_xy_min')
_offset_xy_max = tf.minimum(model.offset_xy_max, self.offset_xy_max, name='_offset_xy_max')
_wh = tf.maximum(_offset_xy_max - _offset_xy_min, 0.0, name='_wh')
_areas = tf.reduce_prod(_wh, -1, name='_areas')
areas = tf.maximum(self.areas + model.areas - _areas, 1e-10, name='areas')
iou = tf.truediv(_areas, areas, name=name)
with tf.name_scope('mask'):
best_box_iou = tf.reduce_max(iou, 2, True, name='best_box_iou')
best_box = tf.to_float(tf.equal(iou, best_box_iou), name='best_box')
mask_best = tf.identity(self.mask * best_box, name='mask_best')
mask_normal = tf.identity(1 - mask_best, name='mask_normal')
with tf.name_scope('dist'):
iou_dist = tf.square(model.iou - mask_best, name='iou_dist')
coords_dist = tf.square(model.coords - self.coords, name='coords_dist')
prob_dist = tf.square(model.prob - self.prob, name='prob_dist')
with tf.name_scope('objectives'):
cnt = np.multiply.reduce(iou_dist.get_shape().as_list())
self['iou_best'] = tf.identity(tf.reduce_sum(mask_best * iou_dist) / cnt, name='iou_best')
self['iou_normal'] = tf.identity(tf.reduce_sum(mask_normal * iou_dist) / cnt, name='iou_normal')
self['coords'] = tf.identity(tf.reduce_sum(tf.expand_dims(mask_best, -1) * coords_dist) / cnt, name='coords')
self['prob'] = tf.identity(tf.reduce_sum(tf.expand_dims(self.mask, -1) * prob_dist) / cnt, name='prob')
def answerer(self, embeddings, question):
embs_after_reading_mat = tf.tile(tf.reshape(embeddings, [self.n_data, 1, 1, self.rank]),
[1, self.n_labels, 1, 1])
fixed_embs = tf.reshape(tf.gather(self.emb0, question[:, :, 1]), [self.n_data, self.n_labels, 1, self.rank])
emb1_question = tf.concat(2, [fixed_embs, embs_after_reading_mat])
preds = tf.reduce_sum(tf.reduce_prod(emb1_question, 2), 2)
return preds
def _expectation(p, kern, feat, none1, none2, nghp=None):
"""
Compute the expectation:
<K_{X, Z}>_p(X)
- K_{.,.} :: RBF kernel
:return: NxM
"""
with params_as_tensors_for(kern), params_as_tensors_for(feat):
# use only active dimensions
Xcov = kern._slice_cov(p.cov)
Z, Xmu = kern._slice(feat.Z, p.mu)
D = tf.shape(Xmu)[1]
if kern.ARD:
lengthscales = kern.lengthscales
else:
lengthscales = tf.zeros((D,), dtype=settings.tf_float) + kern.lengthscales
chol_L_plus_Xcov = tf.cholesky(tf.matrix_diag(lengthscales ** 2) + Xcov) # NxDxD
all_diffs = tf.transpose(Z) - tf.expand_dims(Xmu, 2) # NxDxM
exponent_mahalanobis = tf.matrix_triangular_solve(chol_L_plus_Xcov, all_diffs, lower=True) # NxDxM
exponent_mahalanobis = tf.reduce_sum(tf.square(exponent_mahalanobis), 1) # NxM
exponent_mahalanobis = tf.exp(-0.5 * exponent_mahalanobis) # NxM
sqrt_det_L = tf.reduce_prod(lengthscales)
sqrt_det_L_plus_Xcov = tf.exp(tf.reduce_sum(tf.log(tf.matrix_diag_part(chol_L_plus_Xcov)), axis=1))
determinants = sqrt_det_L / sqrt_det_L_plus_Xcov # N
return kern.variance * (determinants[:, None] * exponent_mahalanobis)
def _expectation(p, mean, none, kern, feat, nghp=None):
"""
Compute the expectation:
expectation[n] = <x_n K_{x_n, Z}>_p(x_n)
- K_{.,.} :: RBF kernel
:return: NxDxM
"""
Xmu, Xcov = p.mu, p.cov
with tf.control_dependencies([tf.assert_equal(
tf.shape(Xmu)[1], tf.constant(kern.input_dim, settings.tf_int),
message="Currently cannot handle slicing in exKxz.")]):
Xmu = tf.identity(Xmu)
with params_as_tensors_for(kern), params_as_tensors_for(feat):
D = tf.shape(Xmu)[1]
lengthscales = kern.lengthscales if kern.ARD \
else tf.zeros((D,), dtype=settings.float_type) + kern.lengthscales
chol_L_plus_Xcov = tf.cholesky(tf.matrix_diag(lengthscales ** 2) + Xcov) # NxDxD
all_diffs = tf.transpose(feat.Z) - tf.expand_dims(Xmu, 2) # NxDxM
sqrt_det_L = tf.reduce_prod(lengthscales)
sqrt_det_L_plus_Xcov = tf.exp(tf.reduce_sum(tf.log(tf.matrix_diag_part(chol_L_plus_Xcov)), axis=1))
determinants = sqrt_det_L / sqrt_det_L_plus_Xcov # N
exponent_mahalanobis = tf.cholesky_solve(chol_L_plus_Xcov, all_diffs) # NxDxM
non_exponent_term = tf.matmul(Xcov, exponent_mahalanobis, transpose_a=True)
non_exponent_term = tf.expand_dims(Xmu, 2) + non_exponent_term # NxDxM
exponent_mahalanobis = tf.reduce_sum(all_diffs * exponent_mahalanobis, 1) # NxM
exponent_mahalanobis = tf.exp(-0.5 * exponent_mahalanobis) # NxM
return kern.variance * (determinants[:, None] * exponent_mahalanobis)[:, None, :] * non_exponent_term
def entropy(self, mean=None, cov=1):
"""
Note entropy does not depend on its mean.
Arguments
----------
mean: tf.Tensor, optional
vector. Defaults to zero mean.
cov: tf.Tensor, optional
vector or matrix. Defaults to identity.
Returns
-------
tf.Tensor
scalar
"""
if cov == 1:
d = 1
det_cov = 1.0
else:
cov = tf.cast(cov, dtype=tf.float32)
d = get_dims(cov)[0]
if len(cov.get_shape()) == 1:
det_cov = tf.reduce_prod(cov)
else:
det_cov = tf.matrix_determinant(cov)
return 0.5 * (d + d*tf.log(2*np.pi) + tf.log(det_cov))
def __init__(self, n_inputs, n_rules, learning_rate=1e-2):
self.n = n_inputs
self.m = n_rules
self.inputs = tf.placeholder(tf.float32, shape=(None, n_inputs)) # Input
self.targets = tf.placeholder(tf.float32, shape=None) # Desired output
mu = tf.get_variable("mu", [n_rules * n_inputs],
initializer=tf.random_normal_initializer(0, 1)) # Means of Gaussian MFS
sigma = tf.get_variable("sigma", [n_rules * n_inputs],
initializer=tf.random_normal_initializer(0, 1)) # Standard deviations of Gaussian MFS
y = tf.get_variable("y", [1, n_rules], initializer=tf.random_normal_initializer(0, 1)) # Sequent centers
self.params = tf.trainable_variables()
self.rul = tf.reduce_prod(
tf.reshape(tf.exp(-0.5 * tf.square(tf.subtract(tf.tile(self.inputs, (1, n_rules)), mu)) / tf.square(sigma)),
(-1, n_rules, n_inputs)), axis=2) # Rule activations
# Fuzzy base expansion function:
num = tf.reduce_sum(tf.multiply(self.rul, y), axis=1)
den = tf.clip_by_value(tf.reduce_sum(self.rul, axis=1), 1e-12, 1e12)
self.out = tf.divide(num, den)
self.loss = tf.losses.huber_loss(self.targets, self.out) # Loss function computation
# Other loss functions for regression, uncomment to try them:
# loss = tf.sqrt(tf.losses.mean_squared_error(target, out))
# loss = tf.losses.absolute_difference(target, out)
self.optimize = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss) # Optimization step
# Other optimizers, uncomment to try them:
# self.optimize = tf.train.RMSPropOptimizer(learning_rate=learning_rate).minimize(self.loss)
# self.optimize = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(self.loss)
self.init_variables = tf.global_variables_initializer() # Variable initializer
def prob_is_largest(self, Y, mu, var, gh_x, gh_w):
# work out what the mean and variance is of the indicated latent function.
oh_on = tf.cast(tf.one_hot(tf.reshape(Y, (-1,)), self.num_classes, 1.0, 0.0), float_type)
mu_selected = tf.reduce_sum(oh_on * mu, 1)
var_selected = tf.reduce_sum(oh_on * var, 1)
# generate Gauss Hermite grid
X = tf.reshape(mu_selected, (-1, 1)) + gh_x * tf.reshape(
tf.sqrt(tf.clip_by_value(2.0 * var_selected, 1e-10, np.inf)), (-1, 1)
)
# compute the CDF of the Gaussian between the latent functions and the grid (including the selected function)
dist = (tf.expand_dims(X, 1) - tf.expand_dims(mu, 2)) / tf.expand_dims(
tf.sqrt(tf.clip_by_value(var, 1e-10, np.inf)), 2
)
cdfs = 0.5 * (1.0 + tf.erf(dist / np.sqrt(2.0)))
cdfs = cdfs * (1 - 2e-4) + 1e-4
# blank out all the distances on the selected latent function
oh_off = tf.cast(tf.one_hot(tf.reshape(Y, (-1,)), self.num_classes, 0.0, 1.0), float_type)
cdfs = cdfs * tf.expand_dims(oh_off, 2) + tf.expand_dims(oh_on, 2)
# take the product over the latent functions, and the sum over the GH grid.
return tf.matmul(tf.reduce_prod(cdfs, reduction_indices=[1]), tf.reshape(gh_w / np.sqrt(np.pi), (-1, 1)))
def layers(name, layers=None, gradients=None, activations=None):
def entry(i, x):
return tf.stack([
tf.constant(i - 0.5, shape=(), dtype=tf.float32),
tf.constant(i + 0.5, shape=(), dtype=tf.float32),
tf.cast(x, dtype=tf.float32)
])
with tf.name_scope(name):
summary_list = []
# weight size summary
if layers is not None:
sizes_entries = []
for i, l in enumerate(layers):
size = sum([tf.reduce_prod(tf.shape(v)) for v in l.variables])
sizes_entries.append(entry(i, size))
sizes_tensor = tf.stack(sizes_entries)
total_size = np.sum(
[np.prod(v.shape) for l in layers for v in l.variables])
metadata = create_summary_metadata(
display_name='sizes',
description='total: {}'.format(total_size))
sizes_summary = tf.summary.tensor_summary(
'sizes', sizes_tensor, summary_metadata=metadata)
summary_list.append(sizes_summary)
# weight L2 norm summary
if layers is not None:
vars_entries = []
for i, l in enumerate(layers):
vnorm = sum([tf.nn.l2_loss(v) for v in l.variables])
vars_entries.append(entry(i, vnorm))
vars_tensor = tf.stack(vars_entries)
metadata = create_summary_metadata(display_name='weights',
description=None)
vars_summary = tf.summary.tensor_summary('weights',
vars_tensor,
summary_metadata=metadata)
summary_list.append(vars_summary)
# activation L2 norm summary
if activations is not None:
act_entries = []
for i, x in enumerate(activations):
xnorm = tf.nn.l2_loss(x)
act_entries.append(entry(i, xnorm))
act_tensor = tf.stack(act_entries)
metadata = create_summary_metadata(display_name='activations',
description=None)
act_summary = tf.summary.tensor_summary('activations',
act_tensor,
summary_metadata=metadata)
summary_list.append(act_summary)
# gradient L2 norm summary
if layers is not None and gradients is not None:
grads_entries = []
for i, l in enumerate(layers):
glist = [g for g, v in gradients if v in set(l.variables)]
gnorm = sum([tf.nn.l2_loss(g) for g in glist])
grads_entries.append(entry(i, gnorm))
grads_tensor = tf.stack(grads_entries)
metadata = create_summary_metadata(display_name='gradients',
description=None)
grads_summary = tf.summary.tensor_summary(
'gradients', grads_tensor, summary_metadata=metadata)
summary_list.append(grads_summary)
merged_summary = tf.summary.merge(summary_list)
return merged_summary
def zca_normalization(features):
shape = tf.shape(features)
# reshape the features to orderless feature vectors
mean_features = tf.reduce_mean(features, axis=[1, 2], keep_dims=True)
unbiased_features = tf.reshape(features - mean_features,
shape=(shape[0], -1, shape[3]))
#unbiased_features = tf.reshape(features, shape=(shape[0], -1, shape[3]))
# get the covariance matrix
gram = tf.matmul(unbiased_features, unbiased_features, transpose_a=True)
gram /= tf.reduce_prod(tf.cast(shape[1:3], tf.float32))
# converting the feature spaces
s, u, v = tf.svd(gram, compute_uv=True)
#s = tf.expand_dims(s, axis=1) # let it be active in the last dimension
# get the effective singular values
'''
valid_index = tf.cast(s > 0.00001, dtype=tf.float32)
s_effective = tf.maximum(s, 0.00001)
sqrt_s_effective = tf.sqrt(s_effective) * valid_index
sqrt_inv_s_effective = tf.sqrt(1.0/s_effective) * valid_index'''
#k = tf.reduce_sum(tf.cast(tf.greater(S, 1e-5), tf.int32))
# noise
sh = tf.shape(u)
noise = tf.random_normal(shape=(sh[1], sh[2]),
mean=0,
stddev=1,
dtype=tf.float32)
s_n, u_n, v_n = tf.svd(noise, compute_uv=True)
#s=tf.diag(s)
u = tf.reshape(u, shape=(sh[1], sh[2]))
u_n = tf.matmul(u_n, u)
#u_n = tf.matmul(noise, u)
#s=tf.reshape(s,shape=(sh[1],sh[2]))
sqrt_s_effective = tf.diag(tf.pow(s, 0.5))
sqrt_s_effective = tf.reshape(sqrt_s_effective, shape=(sh[1], sh[2]))
sqrt_inv_s_effective = tf.diag(tf.pow(s, -0.5))
sqrt_inv_s_effective = tf.reshape(sqrt_inv_s_effective,
shape=(sh[1], sh[2]))
unbiased_features = tf.reshape(unbiased_features, shape=(-1, sh[2]))
# colorization functions
colorization_kernel_n = tf.matmul(tf.matmul(u_n, sqrt_s_effective),
u,
transpose_b=True)
colorization_kernel = tf.matmul(tf.matmul(u, sqrt_s_effective),
u,
transpose_b=True)
#colorization_kernel_n = tf.random_normal(shape=(tf.shape(colorization_kernel)[0],tf.shape(colorization_kernel)[1]),mean=0,stddev=1,dtype=tf.float32)
ratio_n = 0.5
colorization_kernel = ratio_n * colorization_kernel_n + (
1 - ratio_n) * colorization_kernel
colorization_kernel = tf.expand_dims(colorization_kernel, axis=0)
# normalized features
#normalized_features = tf.matmul(tf.matmul(tf.matmul(u, sqrt_inv_s_effective), u, transpose_b=True), unbiased_features)
normalized_features = tf.matmul(unbiased_features, u)
normalized_features = tf.matmul(normalized_features, sqrt_inv_s_effective)
normalized_features = tf.matmul(normalized_features, u, transpose_b=True)
normalized_features = tf.reshape(normalized_features, shape=shape)
return normalized_features, colorization_kernel, mean_features
def getGaussianDiffEntropy(mean, logstd):
std = tf.reduce_prod(tf.exp(logstd))
diffEnt = 0.5 * tf.log(2 * np.pi * np.exp(1) * (std)**2)
return diffEnt
def testEmptyGradients(self):
with self.test_session():
x = tf.zeros([0, 3])
y = tf.reduce_prod(x, [1])
error = tf.test.compute_gradient_error(x, [0, 3], y, [0])
self.assertEqual(error, 0)
def _get_anchor_free(self, key, boxes, classes, height, width, stride,
center_radius):
"""Find the box assignements in an anchor free paradigm."""
level_limits = self.anchor_free_level_limits[key]
gen = loss_utils.GridGenerator(anchors=[[1, 1]], scale_anchors=stride)
grid_points = gen(width, height, 1, boxes.dtype)[0]
grid_points = tf.squeeze(grid_points, axis=0)
box_list = boxes
class_list = classes
grid_points = (grid_points + 0.5) * stride
x_centers, y_centers = grid_points[..., 0], grid_points[..., 1]
boxes *= (tf.convert_to_tensor([width, height, width, height]) *
stride)
tlbr_boxes = box_ops.xcycwh_to_yxyx(boxes)
boxes = tf.reshape(boxes, [1, 1, -1, 4])
tlbr_boxes = tf.reshape(tlbr_boxes, [1, 1, -1, 4])
if self.use_tie_breaker:
area = tf.reduce_prod(boxes[..., 2:], axis=-1)
# check if the box is in the receptive feild of the this fpn level
b_t = y_centers - tlbr_boxes[..., 0]
b_l = x_centers - tlbr_boxes[..., 1]
b_b = tlbr_boxes[..., 2] - y_centers
b_r = tlbr_boxes[..., 3] - x_centers
box_delta = tf.stack([b_t, b_l, b_b, b_r], axis=-1)
if level_limits is not None:
max_reg_targets_per_im = tf.reduce_max(box_delta, axis=-1)
gt_min = max_reg_targets_per_im >= level_limits[0]
gt_max = max_reg_targets_per_im <= level_limits[1]
is_in_boxes = tf.logical_and(gt_min, gt_max)
else:
is_in_boxes = tf.reduce_min(box_delta, axis=-1) > 0.0
is_in_boxes_all = tf.reduce_any(is_in_boxes,
axis=(0, 1),
keepdims=True)
# check if the center is in the receptive feild of the this fpn level
c_t = y_centers - (boxes[..., 1] - center_radius * stride)
c_l = x_centers - (boxes[..., 0] - center_radius * stride)
c_b = (boxes[..., 1] + center_radius * stride) - y_centers
c_r = (boxes[..., 0] + center_radius * stride) - x_centers
centers_delta = tf.stack([c_t, c_l, c_b, c_r], axis=-1)
is_in_centers = tf.reduce_min(centers_delta, axis=-1) > 0.0
is_in_centers_all = tf.reduce_any(is_in_centers,
axis=(0, 1),
keepdims=True)
# colate all masks to get the final locations
is_in_index = tf.logical_or(is_in_boxes_all, is_in_centers_all)
is_in_boxes_and_center = tf.logical_and(is_in_boxes, is_in_centers)
is_in_boxes_and_center = tf.logical_and(is_in_index,
is_in_boxes_and_center)
if self.use_tie_breaker:
boxes_all = tf.cast(is_in_boxes_and_center, area.dtype)
boxes_all = ((boxes_all * area) + ((1 - boxes_all) * INF))
boxes_min = tf.reduce_min(boxes_all, axis=-1, keepdims=True)
boxes_min = tf.where(boxes_min == INF, -1.0, boxes_min)
is_in_boxes_and_center = boxes_all == boxes_min
# construct the index update grid
reps = tf.reduce_sum(tf.cast(is_in_boxes_and_center, tf.int16),
axis=-1)
indexes = tf.cast(tf.where(is_in_boxes_and_center), tf.int32)
y, x, t = tf.split(indexes, 3, axis=-1)
boxes = tf.gather_nd(box_list, t)
classes = tf.cast(tf.gather_nd(class_list, t), boxes.dtype)
reps = tf.gather_nd(reps, tf.concat([y, x], axis=-1))
reps = tf.cast(tf.expand_dims(reps, axis=-1), boxes.dtype)
classes = tf.cast(tf.expand_dims(classes, axis=-1), boxes.dtype)
conf = tf.ones_like(classes)
# return the samples and the indexes
samples = tf.concat([boxes, conf, classes], axis=-1)
indexes = tf.concat([y, x, tf.zeros_like(t)], axis=-1)
return indexes, samples
def tensor_size(tensor): s = tf.shape(tensor) return tf.reduce_prod(s[1:])
def __init__(self,
asset_number,
feature_network_topology,
action_network_layers=[64, 128],
object_function='sortino',
learning_rate=0.001):
tf.reset_default_graph()
self.real_asset_number = asset_number + 1
self.z = tf.placeholder(dtype=tf.float32,
shape=[None, self.real_asset_number],
name='environment_return')
self.c = tf.placeholder(dtype=tf.float32,
shape=[],
name='environment_fee')
self.dropout_keep_prob = tf.placeholder(dtype=tf.float32,
shape=[],
name='dropout_keep_prob')
self.tao = tf.placeholder(dtype=tf.float32,
shape=[],
name='action_temperature')
self.model_inputs = {}
self.feature_outputs = []
self.keep_output = None
self.concat_factor = []
for k, v in feature_network_topology.items():
with tf.variable_scope(
k,
initializer=tf.contrib.layers.xavier_initializer(
uniform=False)):
X = tf.placeholder(
dtype=tf.float32,
shape=[v['feature_map_number'], None, v['feature_number']],
name=v['input_name'])
self.model_inputs[k] = X
output = tl.layers.normalization.batch_normalization(X)
if 'dense' in v:
dense_config = v['dense']
for n, a in zip(dense_config['n_units'],
dense_config['act']):
output = self._add_dense_layer(
output,
output_shape=n,
drop_keep_prob=self.dropout_keep_prob,
act=a)
output = tl.layers.normalization.batch_normalization(
output)
if 'rnn' in v:
rnn_config = v['rnn']
rnn_cells = [
self._add_letm_cell(i, a) for i, a in list(
zip(rnn_config['n_units'], rnn_config['act']))
]
layered_cell = tf.contrib.rnn.MultiRNNCell(rnn_cells)
attention = tf.contrib.rnn.AttentionCellWrapper(
cell=layered_cell,
attn_length=rnn_config['attention_length'])
output, state = tf.nn.dynamic_rnn(cell=attention,
inputs=output,
dtype=tf.float32)
output = tf.unstack(output, axis=0)
if v['feature_map_number'] > 1:
output = tl.layers.merge(output, mode='concat')
else:
output = output[0]
output = tf.concat(
(tf.zeros(shape=[1, output.shape[1]]), output), axis=0)
output = tl.layers.normalization.batch_normalization(
output)
self.feature_outputs.append(output)
if v['keep_output']:
self.keep_output = output
with tf.variable_scope(
'action',
initializer=tf.contrib.layers.xavier_initializer(
uniform=False)):
feature_maps = tl.layers.merge(self.feature_outputs, mode='concat')
for l in action_network_layers:
feature_maps = self._add_dense_layer(feature_maps, l,
self.dropout_keep_prob)
feature_maps = self._add_dense_layer(feature_maps,
self.real_asset_number,
self.dropout_keep_prob,
act=tf.nn.sigmoid)
feature_maps = tl.layers.normalization.batch_normalization(
feature_maps)
cash_vector = self._add_dense_layer(feature_maps,
1,
self.dropout_keep_prob,
act=None)
self.action = tl.layers.merge([self.keep_output, cash_vector],
mode='concat')
self.action = tl.layers.merge([self.action, feature_maps],
mode='elemwise_mul')
self.action = self.action / self.tao
self.action = tf.nn.softmax(self.action)
with tf.variable_scope('reward'):
self.reward_t = tf.reduce_sum(
self.z * self.action[:-1] -
self.c * tf.abs(self.action[1:] - self.action[:-1]),
axis=1)
self.log_reward_t = tf.log(self.reward_t)
self.cum_reward = tf.reduce_prod(self.reward_t)
self.cum_log_reward = tf.reduce_sum(self.log_reward_t)
self.mean_log_reward = tf.reduce_mean(self.log_reward_t)
self.sortino = self._sortino_ratio(self.log_reward_t, 0)
self.sharpe = self._sharpe_ratio(self.log_reward_t, 0)
with tf.variable_scope('train'):
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
if object_function == 'reward':
self.train_op = optimizer.minimize(-self.mean_log_reward)
elif object_function == 'sharpe':
self.train_op = optimizer.minimize(-self.sharpe)
else:
self.train_op = optimizer.minimize(-self.sortino)
self.init_op = tf.global_variables_initializer()
self.saver = tf.train.Saver()
self.session = tf.Session()
def desc_loss(descriptors, warped_descriptors, homographies, valid_mask=None):
"""
"""
positive_margin = tf.constant(1.)
negative_margin = tf.constant(0.2)
lambda_d = tf.constant(0.05) # 800
# Compute the position of the center pixel of every cell in the image
(batch_size, Hc, Wc) = tf.unstack(tf.cast(tf.shape(descriptors)[:3], dtype=tf.int32))
coord_cells = tf.stack(tf.meshgrid(tf.range(Hc), tf.range(Wc), indexing='ij'), axis=-1)
coord_cells = coord_cells * 8 + 8 // 2 # (Hc, Wc, 2)
# coord_cells is now a grid containing the coordinates of the Hc x Wc
# center pixels of the 8x8 cells of the image
# Compute the position of the warped center pixels
warped_coord_cells = warp_points(tf.reshape(coord_cells, [-1, 2]), homographies)
# warped_coord_cells is now a list of the warped coordinates of all the center
# pixels of the 8x8 cells of the image, shape (N, Hc x Wc, 2)
# Compute the pairwise distances and filter the ones less than a threshold
# The distance is just the pairwise norm of the difference of the two grids
# Using shape broadcasting, cell_distances has shape (N, Hc, Wc, Hc, Wc)
coord_cells = tf.cast(tf.reshape(coord_cells, [1, 1, 1, Hc, Wc, 2]), dtype=tf.float32)
warped_coord_cells = tf.reshape(warped_coord_cells, [batch_size, Hc, Wc, 1, 1, 2])
cell_distances = tf.norm(coord_cells - warped_coord_cells, axis=-1)
s = tf.cast(tf.less_equal(cell_distances, 8 - 0.5), dtype=tf.float32)
# s[id_batch, h, w, h', w'] == 1 if the point of coordinates (h, w) warped by the
# homography is at a distance from (h', w') less than config['grid_size']
# and 0 otherwise
# Compute the pairwise dot product between descriptors: d^t * d'
descriptors = tf.reshape(descriptors, [batch_size, Hc, Wc, 1, 1, -1])
descriptors = tf.nn.l2_normalize(descriptors, -1)
warped_descriptors = tf.reshape(warped_descriptors, [batch_size, 1, 1, Hc, Wc, -1])
warped_descriptors = tf.nn.l2_normalize(warped_descriptors, -1)
dot_product_desc = tf.reduce_sum(descriptors * warped_descriptors, -1)
dot_product_desc = tf.nn.relu(dot_product_desc)
dot_product_desc = tf.reshape(tf.nn.l2_normalize(tf.reshape(dot_product_desc, [batch_size, Hc, Wc, Hc * Wc]),
3), [batch_size, Hc, Wc, Hc, Wc])
dot_product_desc = tf.reshape(tf.nn.l2_normalize(tf.reshape(dot_product_desc, [batch_size, Hc * Wc, Hc, Wc]),
1), [batch_size, Hc, Wc, Hc, Wc])
# dot_product_desc[id_batch, h, w, h', w'] is the dot product between the
# descriptor at position (h, w) in the original descriptors map and the
# descriptor at position (h', w') in the warped image
# Compute the loss
positive_dist = tf.maximum(0., positive_margin - dot_product_desc)
negative_dist = tf.maximum(0., dot_product_desc - negative_margin)
loss = lambda_d * s * positive_dist + (1 - s) * negative_dist
# Mask the pixels if bordering artifacts appear
valid_mask = tf.cast(valid_mask[..., tf.newaxis], dtype=tf.float32) # for GPU
valid_mask = tf.space_to_depth(valid_mask, block_size=8)
valid_mask = tf.reduce_prod(valid_mask, axis=3) # AND along the channel dim
valid_mask = tf.reshape(valid_mask, [batch_size, 1, 1, Hc, Wc])
normalization = tf.reduce_sum(valid_mask) * tf.cast(Hc * Wc, dtype=tf.float32)
loss_desc = tf.reduce_sum(valid_mask * loss) / normalization
return loss_desc
def D_main(
images_in, # First input: Images [minibatch, channel, height, width].
labels_in, # Second input: Conditioning labels [minibatch, label_size].
# Input dimensions.
num_channels=3, # Number of input color channels. Overridden based on dataset.
resolution=1024, # Input resolution. Overridden based on dataset.
label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
# Capacity.
fmap_base=16384, # Overall multiplier for the number of feature maps.
fmap_decay=1, # Log2 feature map reduction when doubling the resolution.
fmap_min=1, # Minimum number of feature maps in any layer.
fmap_max=512, # Maximum number of feature maps in any layer.
# Internal details.
mapping_layers=0, # Number of additional mapping layers for the conditioning labels.
mapping_fmaps=None, # Number of activations in the mapping layers, None = default.
mapping_lrmul=0.1, # Learning rate multiplier for the mapping layers.
architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'.
nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', etc.
mbstd_group_size=None, # Group size for the minibatch standard deviation layer, None = entire minibatch.
mbstd_num_features=1, # Number of features for the minibatch standard deviation layer, 0 = disable.
dtype='float32', # Data type to use for intermediate activations and outputs.
num_fp16_res=0, # Use FP16 for the N highest resolutions, regardless of dtype.
conv_clamp=None, # Clamp the output of convolution layers to [-conv_clamp, +conv_clamp], None = disable clamping.
resample_kernel=[
1, 3, 3, 1
], # Low-pass filter to apply when resampling activations, None = box filter.
# Comparison methods.
augment_strength=0, # AdaptiveAugment.get_strength_var() for pagan & adropout.
use_pagan=False, # pagan: Enable?
pagan_num=16, # pagan: Number of active bits with augment_strength=1.
pagan_fade=0.5, # pagan: Relative duration of fading in new bits.
score_size=1, # auxrot: Number of scalars to output. Can vary between evaluations.
score_max=1, # auxrot: Maximum number of scalars to output. Must be set at construction time.
use_spectral_norm=False, # spectralnorm: Enable?
adaptive_dropout=0, # adropout: Standard deviation to use with augment_strength=1, 0 = disable.
freeze_layers=0, # Freeze-D: Number of layers to freeze.
**_kwargs, # Ignore unrecognized keyword args.
):
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return np.clip(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_min,
fmap_max)
assert architecture in ['orig', 'skip', 'resnet']
if mapping_fmaps is None:
mapping_fmaps = nf(0)
act = nonlinearity
# Inputs.
images_in.set_shape([None, num_channels, resolution, resolution])
labels_in.set_shape([None, label_size])
images_in = tf.cast(images_in, dtype)
labels_in = tf.cast(labels_in, dtype)
# Label embedding and mapping.
if label_size > 0:
y = labels_in
with tf.variable_scope('LabelEmbed'):
y = apply_bias_act(dense_layer(y, fmaps=mapping_fmaps))
y = normalize_2nd_moment(y)
for idx in range(mapping_layers):
with tf.variable_scope(f'Mapping{idx}'):
y = apply_bias_act(dense_layer(y,
fmaps=mapping_fmaps,
lrmul=mapping_lrmul),
act=act,
lrmul=mapping_lrmul)
labels_in = y
# Adaptive multiplicative dropout.
def adrop(x):
if adaptive_dropout != 0:
s = [tf.shape(x)[0], x.shape[1]] + [1] * (x.shape.rank - 2)
x *= tf.cast(
tf.exp(
tf.random_normal(s) *
(augment_strength * adaptive_dropout)), x.dtype)
return x
# Freeze-D.
cur_layer_idx = 0
def is_next_layer_trainable():
nonlocal cur_layer_idx
trainable = (cur_layer_idx >= freeze_layers)
cur_layer_idx += 1
return trainable
# Construct PA-GAN bit vector.
pagan_bits = None
pagan_signs = None
if use_pagan:
with tf.variable_scope('PAGAN'):
idx = tf.range(pagan_num, dtype=tf.float32)
active = (augment_strength * pagan_num - idx - 1) / max(
pagan_fade, 1e-8) + 1
prob = tf.clip_by_value(active[np.newaxis, :], 0, 1) * 0.5
rnd = tf.random_uniform([tf.shape(images_in)[0], pagan_num])
pagan_bits = tf.cast(rnd < prob, dtype=tf.float32)
pagan_signs = tf.reduce_prod(1 - pagan_bits * 2,
axis=1,
keepdims=True)
# FromRGB block.
def fromrgb(x, y, res): # res = 2..resolution_log2
with tf.variable_scope('FromRGB'):
trainable = is_next_layer_trainable()
t = tf.cast(
y,
'float16' if res > resolution_log2 - num_fp16_res else dtype)
t = adrop(
conv2d_layer(t,
fmaps=nf(res - 1),
kernel=1,
trainable=trainable))
if pagan_bits is not None:
with tf.variable_scope('PAGAN'):
t += dense_layer(tf.cast(pagan_bits, t.dtype),
fmaps=nf(res - 1),
trainable=trainable)[:, :, np.newaxis,
np.newaxis]
t = apply_bias_act(t,
act=act,
clamp=conv_clamp,
trainable=trainable)
if x is not None:
t += tf.cast(x, t.dtype)
return t
# Main block for one resolution.
def block(x, res): # res = 2..resolution_log2
x = tf.cast(
x, 'float16' if res > resolution_log2 - num_fp16_res else dtype)
t = x
with tf.variable_scope('Conv0'):
trainable = is_next_layer_trainable()
x = apply_bias_act(adrop(
conv2d_layer(x,
fmaps=nf(res - 1),
kernel=3,
trainable=trainable,
use_spectral_norm=use_spectral_norm)),
act=act,
clamp=conv_clamp,
trainable=trainable)
with tf.variable_scope('Conv1_down'):
trainable = is_next_layer_trainable()
x = apply_bias_act(adrop(
conv2d_layer(x,
fmaps=nf(res - 2),
kernel=3,
down=True,
resample_kernel=resample_kernel,
trainable=trainable,
use_spectral_norm=use_spectral_norm)),
act=act,
clamp=conv_clamp,
trainable=trainable)
if architecture == 'resnet':
with tf.variable_scope('Skip'):
trainable = is_next_layer_trainable()
t = adrop(
conv2d_layer(t,
fmaps=nf(res - 2),
kernel=1,
down=True,
resample_kernel=resample_kernel,
trainable=trainable))
x = (x + t) * (1 / np.sqrt(2))
return x
# Downsampling block.
def downsample(y):
with tf.variable_scope('Downsample'):
return downsample_2d(y, k=resample_kernel)
# Layers for >=8x8 resolutions.
x = None
y = images_in
for res in range(resolution_log2, 2, -1):
with tf.variable_scope(f'{2**res}x{2**res}'):
if architecture == 'skip' or res == resolution_log2:
x = fromrgb(x, y, res)
x = block(x, res)
if architecture == 'skip':
y = downsample(y)
# Layers for 4x4 resolution.
with tf.variable_scope('4x4'):
if architecture == 'skip':
x = fromrgb(x, y, 2)
x = tf.cast(x, dtype)
if mbstd_num_features > 0:
with tf.variable_scope('MinibatchStddev'):
x = minibatch_stddev_layer(x, mbstd_group_size,
mbstd_num_features)
with tf.variable_scope('Conv'):
trainable = is_next_layer_trainable()
x = apply_bias_act(adrop(
conv2d_layer(x,
fmaps=nf(1),
kernel=3,
trainable=trainable,
use_spectral_norm=use_spectral_norm)),
act=act,
clamp=conv_clamp,
trainable=trainable)
with tf.variable_scope('Dense0'):
trainable = is_next_layer_trainable()
x = apply_bias_act(adrop(
dense_layer(x, fmaps=nf(0), trainable=trainable)),
act=act,
trainable=trainable)
# Output layer (always trainable).
with tf.variable_scope('Output'):
if label_size > 0:
assert score_max == 1
x = apply_bias_act(dense_layer(x, fmaps=mapping_fmaps))
x = tf.reduce_sum(x * labels_in, axis=1,
keepdims=True) / np.sqrt(mapping_fmaps)
else:
x = apply_bias_act(dense_layer(x, fmaps=score_max))
if pagan_signs is not None:
assert score_max == 1
x *= pagan_signs
scores_out = x[:, :score_size]
# Output.
assert scores_out.dtype == tf.as_dtype(dtype)
scores_out = tf.identity(scores_out, name='scores_out')
return scores_out
def run():
if tf.__version__.split('.')[0] != "1":
raise Exception("Tensorflow version 1 required")
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"ngf", "ndf"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
model = Model(a, loader.room_nc, loader.room_width)
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar('predict_fake', tf.squeeze(model.predict_fake))
tf.summary.scalar('predict_real', tf.squeeze(model.predict_real))
# tf.summary.scalar("discriminator_gan_loss", model.discrim_GAN_loss)
# tf.summary.scalar('discriminator_grad_pen', model.grad_pen)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
tf.summary.scalar('category_loss', model.category_loss)
tf.summary.scalar('rotation_loss', model.rotation_loss)
tf.summary.scalar("generator_adversarial_loss", model.gen_loss)
tf.summary.scalar("generator_supervised_loss", model.gen_supervised_loss)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=100)
a.summary_freq *= loader.ntrain
a.progress_freq *= loader.ntrain
a.save_freq *= loader.ntrain
a.display_freq *= loader.ntrain
a.validate_freq *= loader.ntrain
logdir = a.output_dir if a.summary_freq > 0 else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
# checkpoint = tf.train.latest_checkpoint(a.checkpoint)
checkpoint = os.path.join(a.checkpoint, a.check_step)
saver.restore(sess, checkpoint)
if a.mode == "test":
validate(-1, model, sess)
else:
max_steps = a.max_epochs * loader.ntrain
for e in range(a.pre_train_G_epoch):
if e % 10 == 0 or e == a.pre_train_G_epoch - 1:
fetches = {"inputs": model.inputs,
'outputs': model.outputs,
'targets': model.targets}
for step in range(loader.ntrain):
results = model.pre_train_G(fetches, sess, loader, 1)
prefix = -a.pre_train_G_epoch + e
append('train.html', results, {
'outputs': '%010d_%03d_outputs.jpg' % (prefix, step),
'targets': '%010d_%03d_targets.jpg' % (prefix, step),
'inputs': '%010d_%03d_inputs.jpg' % (prefix, step),
'step': prefix
})
else:
model.pre_train_G({}, sess, loader, loader.ntrain)
model.pre_train_D({}, sess, loader, a.pre_train_D_epoch * loader.ntrain)
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
fetches = {
"train": model.discrim_train,
}
for _ in range(a.discrim_freq):
rooms, layouts = loader.next_batch(0)
sess.run(fetches, {model.inputs: rooms, model.targets: layouts})
fetches = {
"train": model.gen_adversarial_train,
"global_step": sv.global_step,
}
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["inputs"] = model.inputs
fetches["outputs"] = model.outputs
fetches["targets"] = model.targets
rooms, layouts = loader.next_batch(0)
results = sess.run(fetches, {model.inputs: rooms, model.targets: layouts})
if should(a.validate_freq):
validate(step, model, sess)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
append('train.html', results, {
'step': step,
'outputs': '%010d_outputs.jpg' % step,
'targets': '%010d_targets.jpg' % step,
'inputs': '%010d_inputs.jpg' % step
})
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = (step + 1) / loader.ntrain
train_step = (step + 1) % loader.ntrain
rate = (step + 1) / (time.time() - start)
remaining = (max_steps - step - 1) / rate
print("progress epoch %d remaining %dm" % (train_epoch, remaining / 60))
if should(a.save_freq):
print("saving model")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
def descriptor_loss(descriptors,
warped_descriptors,
homographies,
valid_mask=None,
**config):
# Compute the position of the center pixel of every cell in the image
(batch_size, Hc, Wc) = tf.unstack(tf.to_int32(tf.shape(descriptors)[:3]))
coord_cells = tf.stack(tf.meshgrid(tf.range(Hc),
tf.range(Wc),
indexing='ij'),
axis=-1)
coord_cells = coord_cells * config['grid_size'] + config[
'grid_size'] // 2 # (Hc, Wc, 2)
# coord_cells is now a grid containing the coordinates of the Hc x Wc
# center pixels of the 8x8 cells of the image
# Compute the position of the warped center pixels
warped_coord_cells = warp_points(tf.reshape(coord_cells, [-1, 2]),
homographies)
# warped_coord_cells is now a list of the warped coordinates of all the center
# pixels of the 8x8 cells of the image, shape (N, Hc x Wc, 2)
# Compute the pairwise distances and filter the ones less than a threshold
# The distance is just the pairwise norm of the difference of the two grids
# Using shape broadcasting, cell_distances has shape (N, Hc, Wc, Hc, Wc)
coord_cells = tf.to_float(tf.reshape(coord_cells, [1, 1, 1, Hc, Wc, 2]))
warped_coord_cells = tf.reshape(warped_coord_cells,
[batch_size, Hc, Wc, 1, 1, 2])
cell_distances = tf.norm(coord_cells - warped_coord_cells, axis=-1)
s = tf.to_float(tf.less_equal(cell_distances, config['grid_size'] - 0.5))
# s[id_batch, h, w, h', w'] == 1 if the point of coordinates (h, w) warped by the
# homography is at a distance from (h', w') less than config['grid_size']
# and 0 otherwise
# Normalize the descriptors and
# compute the pairwise dot product between descriptors: d^t * d'
descriptors = tf.reshape(descriptors, [batch_size, Hc, Wc, 1, 1, -1])
descriptors = tf.nn.l2_normalize(descriptors, -1)
warped_descriptors = tf.reshape(warped_descriptors,
[batch_size, 1, 1, Hc, Wc, -1])
warped_descriptors = tf.nn.l2_normalize(warped_descriptors, -1)
dot_product_desc = tf.reduce_sum(descriptors * warped_descriptors, -1)
dot_product_desc = tf.nn.relu(dot_product_desc)
dot_product_desc = tf.reshape(
tf.nn.l2_normalize(
tf.reshape(dot_product_desc, [batch_size, Hc, Wc, Hc * Wc]), 3),
[batch_size, Hc, Wc, Hc, Wc])
dot_product_desc = tf.reshape(
tf.nn.l2_normalize(
tf.reshape(dot_product_desc, [batch_size, Hc * Wc, Hc, Wc]), 1),
[batch_size, Hc, Wc, Hc, Wc])
# dot_product_desc[id_batch, h, w, h', w'] is the dot product between the
# descriptor at position (h, w) in the original descriptors map and the
# descriptor at position (h', w') in the warped image
# Compute the loss
positive_dist = tf.maximum(0.,
config['positive_margin'] - dot_product_desc)
negative_dist = tf.maximum(0.,
dot_product_desc - config['negative_margin'])
loss = config['lambda_d'] * s * positive_dist + (1 - s) * negative_dist
# Mask the pixels if bordering artifacts appear
valid_mask = tf.ones([batch_size,
Hc * config['grid_size'],
Wc * config['grid_size']], tf.float32)\
if valid_mask is None else valid_mask
valid_mask = tf.to_float(valid_mask[..., tf.newaxis]) # for GPU
valid_mask = tf.space_to_depth(valid_mask, config['grid_size'])
valid_mask = tf.reduce_prod(valid_mask,
axis=3) # AND along the channel dim
valid_mask = tf.reshape(valid_mask, [batch_size, 1, 1, Hc, Wc])
normalization = tf.reduce_sum(valid_mask) * tf.to_float(Hc * Wc)
# Summaries for debugging
# tf.summary.scalar('nb_positive', tf.reduce_sum(valid_mask * s) / normalization)
# tf.summary.scalar('nb_negative', tf.reduce_sum(valid_mask * (1 - s)) / normalization)
tf.summary.scalar(
'positive_dist',
tf.reduce_sum(valid_mask * config['lambda_d'] * s * positive_dist) /
normalization)
tf.summary.scalar(
'negative_dist',
tf.reduce_sum(valid_mask * (1 - s) * negative_dist) / normalization)
loss = tf.reduce_sum(valid_mask * loss) / normalization
return loss
def train(self, opt):
opt.num_source = opt.seq_length - 1
# TODO: currently fixed to 4
opt.num_scales = 4
self.opt = opt
self.build_train_graph()
self.collect_summaries()
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) \
for v in tf.trainable_variables()])
self.saver = tf.train.Saver([var for var in tf.model_variables()] + \
[self.global_step],
max_to_keep=10)
sv = tf.train.Supervisor(logdir=opt.checkpoint_dir,
save_summaries_secs=0,
saver=None)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with sv.managed_session(config=config) as sess:
print('Trainable variables: ')
for var in tf.trainable_variables():
print(var.name)
print("parameter_count =", sess.run(parameter_count))
#print(tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS,
# scope='depth_prediciton')) ### 查看pre_depth变量名 3/21 打印为空[]
if opt.continue_train:
if opt.init_checkpoint_file is None:
checkpoint = tf.train.latest_checkpoint(opt.checkpoint_dir)
else:
checkpoint = opt.init_checkpoint_file
print("Resume training from previous checkpoint: %s" % checkpoint)
self.saver.restore(sess, checkpoint)
start_time = time.time()
for step in range(1, opt.max_steps):
fetches = {
"train": self.train_op,
"global_step": self.global_step,
"incr_global_step": self.incr_global_step,
#"train_intrinsics": self.train_op_intrinsics ### add 3/23 delete 3/23
}
if step % opt.summary_freq == 0:
fetches["loss"] = self.total_loss
fetches["depth"] = self.pred_depth ### add 3/21 delete 3/22 add 3/23
#fetches["pixel_loss"] = self.pixel_loss ### add 3/20
#fetches["exp_loss"] = self.exp_loss ### add 3/20
#fetches["smooth_loss"] = self.smooth_loss ### add 3/20
#fetches["scaling_loss"] = self.scaling_loss ### add 3/20
#fetches["learning_rate_intrinsics"] = self.learning_rate_intrinsics ### 打印学习率 3/23 delete
fetches["summary"] = sv.summary_op
results = sess.run(fetches)
gs = results["global_step"]
if step % opt.summary_freq == 0:
sv.summary_writer.add_summary(results["summary"], gs)
train_epoch = math.ceil(gs / self.steps_per_epoch)
train_step = gs - (train_epoch - 1) * self.steps_per_epoch
print("Epoch: [%2d] [%5d/%5d] time: %4.4f/it loss: %.3f" \
% (train_epoch, train_step, self.steps_per_epoch, \
(time.time() - start_time)/opt.summary_freq,
results["loss"]))
#intrinsics = sess.graph.get_tensor_by_name('data_loading/intrinsics:0') ### add 3/13
#print(sess.run(intrinsics)) ### add 观察相机内参训练情况 3/13
#print()
#var1 = sess.graph.get_tensor_by_name('data_loading/intrinsics_scale:0') ### add 3/14
#var2 = sess.graph.get_tensor_by_name('data_loading/intrinsics_trans:0') ### add 3/14
#print(sess.run(var1)) ### add 观察相机内参训练情况 3/14
#print(sess.run(var2)) ### add 观察相机内参训练情况 3/14
#print("intrinsics learning rate: %f" % results["learning_rate_intrinsics"])### 观察学习率 3/23
#print("mean: %.4f" % np.mean(results["depth"][0])) ### add 3/21
print("max: %.4f min: %.4f mean: %.4f" % \
(results["depth"][0].max(), \
results["depth"][0].min(), \
results["depth"][0].mean())) ### 观察深度信息 3/21 add 3/23
#print("exp_loss : %.4f" % sess.run(results["exp_loss"])) ### 观察各项loss的量级 3/20
#print("pixel_loss : %.4f" % sess.run(self.pixel_loss)) ### 观察各项loss的量级 3/20
#print("smooth_loss : %.4f" % sess.run(self.smooth_loss)) ### 观察各项loss的量级 3/20
#print("scaling_loss : %.4f" % sess.run(self.scaling_loss)) ### 观察各项loss的量级 3/20
print() ### add 3/14
#for var in tf.trainable_variables(): ### add 3/13
# print(var.name)
# if var.name == 'data_loading/intrinsics:0': ### add 3/13
# print(sess.run(var.value))
start_time = time.time()
if step % opt.save_latest_freq == 0:
self.save(sess, opt.checkpoint_dir, 'latest')
if step % self.steps_per_epoch == 0:
self.save(sess, opt.checkpoint_dir, gs)
def tensors_to_item(self, keys_to_tensors):
item = self._handler.tensors_to_item(keys_to_tensors)
return tf.cond(
pred=tf.equal(tf.reduce_prod(tf.shape(item)), 0),
true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors),
false_fn=lambda: item)
'''
Train loop
'''
init = tf.global_variables_initializer()
merge = tf.summary.merge_all()
# saver = tf.train.Saver()
saver = tf.train.Saver()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
#ckpt = tf.train.latest_checkpoint('./result/ssgan_phone_debug.MODE-local_epce-z.ALI_MODE-concat_x.LEN-160.1581765501/','checkpoint')
#saver.restore(session, ckpt)
gen_num = tf.reduce_sum([tf.reduce_prod(tf.shape(t)) for t in gen_params])
ext_num = tf.reduce_sum([tf.reduce_prod(tf.shape(t)) for t in ext_params])
ext_g_num = tf.reduce_sum([tf.reduce_prod(tf.shape(t)) for t in ext_g_params])
disc_num = tf.reduce_sum([tf.reduce_prod(tf.shape(t)) for t in disc_params])
c_num = tf.reduce_sum([tf.reduce_prod(tf.shape(t)) for t in classl_params+classg_params])
print('finish loading model!')
#reconstruct_video(0)
#disentangle(0)
# test_writer = tf.summary.FileWriter(log_dir + '/restore')
# test_accuracy_list = []
# test_loss_list = []
# # test
# for j in tqdm(range(test_total_batch)):
# x_test_batch, y_test
# _batch = get_batch(test_image, test_label, batch_size, j, test_total_batch)
# summary,test_accuracy,test_loss = sess.run([merge, accuracy,loss], feed_dict = {x:x_test_batch, y:y_test_batch, keep_prob:dropout_rate})
def get_page_accurate(self):
c = tf.cast(tf.equal(self.viterbi_sequence, self.targets),
dtype=tf.float32)
d = tf.reduce_prod(c, 1)
return tf.metrics.mean(d)
def main():
if a.seed is None:
a.seed = random.randint(0, 2 ** 31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "features":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputsX, examples.inputsY, a)
# undo colorization splitting on images that we use for display/output
inputsX = deprocess(examples.inputsX)
inputsY = deprocess(examples.inputsY)
outputsX2Y = deprocess(model.outputsX2Y)
outputsY2X = deprocess(model.outputsY2X)
outputsX2Yp = deprocess(model.outputsX2Yp)
outputsY2Xp = deprocess(model.outputsY2Xp)
auto_outputX = deprocess(model.auto_outputX)
auto_outputY = deprocess(model.auto_outputY)
im_swapped_X = deprocess(model.im_swapped_X)
im_swapped_Y = deprocess(model.im_swapped_Y)
sel_auto_X = deprocess(model.sel_auto_X)
sel_auto_Y = deprocess(model.sel_auto_Y)
sR_X2Y = model.sR_X2Y
sR_Y2X = model.sR_Y2X
eR_X2Y = model.eR_X2Y
eR_Y2X = model.eR_Y2X
cycX_output = deprocess(model.cycX_output)
cycY_output = deprocess(model.cycY_output)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputsX"):
converted_inputsX = convert(inputsX)
with tf.name_scope("convert_inputsY"):
converted_inputsY = convert(inputsY)
with tf.name_scope("convert_outputsX2Y"):
converted_outputsX2Y = convert(outputsX2Y)
with tf.name_scope("convert_outputsY2X"):
converted_outputsY2X = convert(outputsY2X)
with tf.name_scope("convert_outputsX2Yp"):
converted_outputsX2Yp = convert(outputsX2Yp)
with tf.name_scope("convert_outputsY2Xp"):
converted_outputsY2Xp = convert(outputsY2Xp)
with tf.name_scope("convert_auto_outputsX"):
converted_auto_outputX = convert(auto_outputX)
with tf.name_scope("convert_auto_outputsY"):
converted_auto_outputY = convert(auto_outputY)
with tf.name_scope("convert_im_swapped_Y"):
converted_im_swapped_Y = convert(im_swapped_Y)
with tf.name_scope("convert_sel_auto_Y"):
converted_sel_auto_Y = convert(sel_auto_Y)
with tf.name_scope("convert_im_swapped_X"):
converted_im_swapped_X = convert(im_swapped_X)
with tf.name_scope("convert_sel_auto_X"):
converted_sel_auto_X = convert(sel_auto_X)
with tf.name_scope("convert_cycX_output"):
converted_cycX_output = convert(cycX_output)
with tf.name_scope("convert_cycY_output"):
converted_cycY_output = convert(cycY_output)
with tf.name_scope("encode_images"):
display_fetches = {
"paths": examples.paths,
"inputsX": tf.map_fn(tf.image.encode_png, converted_inputsX, dtype=tf.string, name="inputX_pngs"),
"inputsY": tf.map_fn(tf.image.encode_png, converted_inputsY, dtype=tf.string, name="inputY_pngs"),
"outputsX2Y": tf.map_fn(tf.image.encode_png, converted_outputsX2Y, dtype=tf.string, name="outputX2Y_pngs"),
"outputsY2X": tf.map_fn(tf.image.encode_png, converted_outputsY2X, dtype=tf.string, name="outputY2X_pngs"),
"outputsX2Yp": tf.map_fn(tf.image.encode_png, converted_outputsX2Yp, dtype=tf.string,
name="outputX2Yp_pngs"),
"outputsY2Xp": tf.map_fn(tf.image.encode_png, converted_outputsY2Xp, dtype=tf.string,
name="outputY2Xp_pngs"),
"auto_outputsX": tf.map_fn(tf.image.encode_png, converted_auto_outputX, dtype=tf.string,
name="auto_outputX_pngs"),
"auto_outputsY": tf.map_fn(tf.image.encode_png, converted_auto_outputY, dtype=tf.string,
name="auto_outputY_pngs"),
"im_swapped_Y": tf.map_fn(tf.image.encode_png, converted_im_swapped_Y, dtype=tf.string,
name="im_swapped_Y_pngs"),
"sel_auto_Y": tf.map_fn(tf.image.encode_png, converted_sel_auto_Y, dtype=tf.string, name="sel_auto_Y_pngs"),
"im_swapped_X": tf.map_fn(tf.image.encode_png, converted_im_swapped_X, dtype=tf.string,
name="im_swapped_X_pngs"),
"sel_auto_X": tf.map_fn(tf.image.encode_png, converted_sel_auto_X, dtype=tf.string, name="sel_auto_X_pngs"),
"cycX_output": tf.map_fn(tf.image.encode_png, converted_cycX_output, dtype=tf.string, name="cycX_output_pngs"),
"cycY_output": tf.map_fn(tf.image.encode_png, converted_cycY_output, dtype=tf.string, name="cycY_output_pngs")
}
with tf.name_scope("extract_features"):
features_fetches = {
"paths": examples.paths,
"inputsX": converted_inputsX,
"sR_X2Y": sR_X2Y,
"eR_X2Y": eR_X2Y,
"inputsY": converted_inputsY,
"sR_Y2X": sR_Y2X,
"eR_Y2X": eR_Y2X,
}
# summaries
with tf.name_scope("X1_input_summary"):
tf.summary.image("inputsX", converted_inputsX, max_outputs=3)
with tf.name_scope("Y1_input_summary"):
tf.summary.image("inputsY", converted_inputsY, max_outputs=3)
with tf.name_scope("X2Y_output_summary"):
tf.summary.image("outputsX2Y", converted_outputsX2Y, max_outputs=3)
with tf.name_scope("Y2X_output2_summary"):
tf.summary.image("outputsY2X", converted_outputsY2X, max_outputs=3)
with tf.name_scope("X_autoencoder_summary"):
tf.summary.image("auto_outputX", converted_auto_outputX, max_outputs=3)
with tf.name_scope("Y_autoencoder_summary"):
tf.summary.image("auto_outputY", converted_auto_outputY, max_outputs=3)
with tf.name_scope("swapped_1Y_summary"):
tf.summary.image("im_swapped_Y", converted_im_swapped_Y, max_outputs=3)
tf.summary.image("sel_auto_Y", converted_sel_auto_Y, max_outputs=3)
with tf.name_scope("swapped_2X_summary"):
tf.summary.image("im_swapped_X", converted_im_swapped_X, max_outputs=3)
tf.summary.image("sel_auto_X", converted_sel_auto_X, max_outputs=3)
with tf.name_scope("otherNoise_output_summary"):
tf.summary.image("outputsX2Yp", converted_outputsX2Yp, max_outputs=3)
tf.summary.image("outputsY2Xp", converted_outputsY2Xp, max_outputs=3)
with tf.name_scope("cycX_output_summary"):
tf.summary.image("cycX_output", converted_cycX_output, max_outputs=3)
with tf.name_scope("cycY_output_summary"):
tf.summary.image("cycY_output", converted_cycY_output, max_outputs=3)
tf.summary.scalar("discriminatorX2Y_loss", model.discrimX2Y_loss)
tf.summary.scalar("discriminatorY2X_loss", model.discrimY2X_loss)
tf.summary.scalar("generatorX2Y_loss", model.genX2Y_loss)
tf.summary.scalar("generatorY2X_loss", model.genY2X_loss)
tf.summary.scalar("discriminator_sharedX2Y_loss", model.discrim_sharedX2Y_loss)
tf.summary.scalar("discriminator_sharedY2X_loss", model.discrim_sharedY2X_loss)
tf.summary.scalar("autoencoderX_loss", model.autoencoderX_loss)
tf.summary.scalar("autoencoderY_loss", model.autoencoderY_loss)
tf.summary.scalar("code_sR_X2Y_recon_loss", model.code_sR_X2Y_recon_loss)
tf.summary.scalar("code_sR_Y2X_recon_loss", model.code_sR_Y2X_recon_loss)
tf.summary.scalar("code_eR_X2Y_recon_loss", model.code_eR_X2Y_recon_loss)
tf.summary.scalar("code_eR_Y2X_recon_loss", model.code_eR_Y2X_recon_loss)
tf.summary.scalar("code_reconX2Y_loss", model.code_reconX2Y_loss)
tf.summary.scalar("code_reconY2X_loss", model.code_reconY2X_loss)
tf.summary.scalar("cycX_loss", model.cycX_loss)
tf.summary.scalar("cycX_loss", model.cycY_loss)
# for var in tf.trainable_variables():
# tf.summary.histogram(var.op.name + "/values", var)
# for grad, var in model.discrimX2Y_grads_and_vars + model.genX2Y_grads_and_vars:
# tf.summary.histogram(var.op.name + "/gradientsX2Y", grad)
# for grad, var in model.discrimY2X_grads_and_vars + model.genY2X_grads_and_vars:
# tf.summary.histogram(var.op.name + "/gradientsY2X", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session(config=tf.ConfigProto(
allow_soft_placement=True, log_device_placement=True)) as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2 ** 31 - 1
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
elif a.mode == "features":
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(features_fetches)
save_features(results)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrimX2Y_loss"] = model.discrimX2Y_loss
fetches["discrimY2X_loss"] = model.discrimY2X_loss
fetches["genX2Y_loss"] = model.genX2Y_loss
fetches["genY2X_loss"] = model.genY2X_loss
fetches["autoencoderX_loss"] = model.autoencoderX_loss
fetches["autoencoderY_loss"] = model.autoencoderY_loss
fetches["code_reconX2Y_loss"] = model.code_reconX2Y_loss
fetches["code_reconY2X_loss"] = model.code_reconY2X_loss
fetches["cycX_loss"] = model.cycX_loss
fetches["cycY_loss"] = model.cycY_loss
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches, options=options, run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"], step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch)
train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print("progress epoch %d step %d image/sec %0.1f remaining %dm" % (
train_epoch, train_step, rate, remaining / 60))
print("discrimX2Y_loss", results["discrimX2Y_loss"])
print("discrimY2X_loss", results["discrimY2X_loss"])
print("genX2Y_loss", results["genX2Y_loss"])
print("genY2X_loss", results["genY2X_loss"])
print("autoencoderX_loss", results["autoencoderX_loss"])
print("autoencoderY_loss", results["autoencoderY_loss"])
print("code_reconX2Y_loss", results["code_reconX2Y_loss"])
print("code_reconY2X_loss", results["code_reconY2X_loss"])
print("cycX_loss", results["cycX_loss"])
print("cycY_loss", results["cycY_loss"])
if should(a.save_freq):
print("saving model")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
def uncertain_conditional(Xnew_mu,
Xnew_var,
feat,
kern,
q_mu,
q_sqrt,
*,
Luu=None,
mean_function=None,
full_output_cov=False,
full_cov=False,
white=False):
"""
Calculates the conditional for uncertain inputs Xnew, p(Xnew) = N(Xnew_mu, Xnew_var).
See ``conditional`` documentation for further reference.
:param Xnew_mu: mean of the inputs, size N x Din
:param Xnew_var: covariance matrix of the inputs, size N x Din x Din
:param feat: gpflow.InducingFeature object, only InducingPoints is supported
:param kern: gpflow kernel or ekernel object.
:param q_mu: mean inducing points, size M x Dout
:param q_sqrt: cholesky of the covariance matrix of the inducing points, size Dout x M x M
:param full_output_cov: boolean wheter to compute covariance between output dimension.
Influences the shape of return value ``fvar``. Default is False
:param white: boolean whether to use whitened representation. Default is False.
:return fmean, fvar: mean and covariance of the conditional, size ``fmean`` is N x Dout,
size ``fvar`` depends on ``full_output_cov``: if True ``f_var`` is N x Dout x Dout,
if False then ``f_var`` is N x Dout
"""
# TODO(VD): Tensorflow 1.7 doesn't support broadcasting in``tf.matmul`` and
# ``tf.matrix_triangular_solve``. This is reported in issue 216.
# As amplitude temporary workaround, we are using ``tf.einsum`` for the matrix
# multiplications and tiling in the triangular solves.
# The code that should be used once the bug is resolved is added in comments.
if not isinstance(feat, InducingPoints):
raise NotImplementedError
if full_cov:
# TODO(VD): ``full_cov`` True would return amplitude ``fvar`` of shape N x N x D x D,
# encoding the covariance between input datapoints as well.
# This is not implemented as this feature is only used for plotting purposes.
raise NotImplementedError
num_data = tf.shape(Xnew_mu)[0] # number of new inputs (N)
num_ind = tf.shape(q_mu)[0] # number of inducing points (M)
num_func = tf.shape(q_mu)[1] # output dimension (D)
q_sqrt_r = tf.matrix_band_part(q_sqrt, -1, 0) # D x M x M
if Luu is None:
Kuu = feat.Kuu(kern, jitter=settings.numerics.jitter_level) # M x M
Luu = tf.cholesky(Kuu) # M x M
if not white:
q_mu = tf.matrix_triangular_solve(Luu, q_mu, lower=True)
Luu_tiled = tf.tile(
Luu[None, :, :],
[num_func, 1, 1]) # remove line once issue 216 is fixed
q_sqrt_r = tf.matrix_triangular_solve(Luu_tiled, q_sqrt_r, lower=True)
lengthscales = kern.lengthscales if kern.ARD \
else tf.zeros((tf.shape(Xnew_mu)[0],), dtype=settings.float_type) + kern.lengthscales
chol_L_plus_Xcov = tf.cholesky(tf.matrix_diag(lengthscales**2) +
Xnew_var) # NxDxD
all_diffs = tf.transpose(feat.Z) - tf.expand_dims(Xnew_mu, 2) # NxDxM
sqrt_det_L = tf.reduce_prod(lengthscales)
sqrt_det_L_plus_Xcov = tf.exp(
tf.reduce_sum(tf.log(tf.matrix_diag_part(chol_L_plus_Xcov)), axis=1))
determinants = sqrt_det_L / sqrt_det_L_plus_Xcov # N
exponent_mahalanobis = tf.cholesky_solve(chol_L_plus_Xcov,
all_diffs) # NxDxM
non_exponent_term = exponent_mahalanobis
#non_exponent_term = tf.matmul(Xnew_var, exponent_mahalanobis, transpose_a=True)
non_exponent_term = tf.expand_dims(Xnew_mu, 2) + non_exponent_term # NxDxM
exponent_mahalanobis = tf.reduce_sum(all_diffs * exponent_mahalanobis,
1) # NxM
exponent_mahalanobis = tf.exp(-0.5 * exponent_mahalanobis) # NxM
eKuf = kern.variance * (determinants[:, None] * exponent_mahalanobis)
exKuf = kern.variance * (determinants[:, None] * exponent_mahalanobis
)[:, None, :] * non_exponent_term
eKuf = tf.transpose(eKuf)
exKuf = tf.transpose(exKuf)
pXnew = Gaussian(Xnew_mu, Xnew_var)
Li_eKuf = tf.matrix_triangular_solve(Luu, eKuf, lower=True) # M x N
Luu_tile = tf.tile(Luu[None, :, :], [tf.shape(Xnew_mu)[1], 1, 1])
Li_exKuf = tf.matrix_triangular_solve(Luu_tile,
tf.transpose(exKuf, [1, 0, 2]),
lower=True) # D x M x N
fmean = tf.matmul(Li_eKuf, q_mu, transpose_a=True)
q_mu_tile = tf.tile(q_mu[None, :, :], [tf.shape(Xnew_mu)[1], 1, 1])
cross_cov_terms = tf.matmul(Li_exKuf, q_mu_tile, transpose_a=True)
cross_cov_terms = tf.transpose(cross_cov_terms, [1, 0, 2])
cross_cov_terms = cross_cov_terms - tf.matmul(
Xnew_mu, fmean, transpose_a=True)
eKff = expectation(pXnew, kern) # N (psi0)
eKuffu = expectation(pXnew, (kern, feat), (kern, feat)) # N x M x M (psi2)
Luu_tiled = tf.tile(
Luu[None, :, :],
[num_data, 1, 1]) # remove this line, once issue 216 is fixed
Li_eKuffu = tf.matrix_triangular_solve(Luu_tiled, eKuffu, lower=True)
Li_eKuffu_Lit = tf.matrix_triangular_solve(Luu_tiled,
tf.matrix_transpose(Li_eKuffu),
lower=True) # N x M x M
cov = tf.matmul(q_sqrt_r, q_sqrt_r, transpose_b=True) # D x M x M
if mean_function is None or isinstance(mean_function, mean_functions.Zero):
e_related_to_mean = tf.zeros((num_data, num_func, num_func),
dtype=settings.float_type)
else:
# Update mean: \mu(x) + m(x)
fmean = fmean + expectation(pXnew, mean_function)
# Calculate: m(x) m(x)^T + m(x) \mu(x)^T + \mu(x) m(x)^T,
# where m(x) is the mean_function and \mu(x) is fmean
e_mean_mean = expectation(pXnew, mean_function,
mean_function) # N x D x D
Lit_q_mu = tf.matrix_triangular_solve(Luu, q_mu, adjoint=True)
e_mean_Kuf = expectation(pXnew, mean_function,
(kern, feat)) # N x D x M
# einsum isn't able to infer the rank of e_mean_Kuf, hence we explicitly set the rank of the tensor:
e_mean_Kuf = tf.reshape(e_mean_Kuf, [num_data, num_func, num_ind])
e_fmean_mean = tf.einsum("nqm,mz->nqz", e_mean_Kuf,
Lit_q_mu) # N x D x D
e_related_to_mean = e_fmean_mean + tf.matrix_transpose(
e_fmean_mean) + e_mean_mean
if full_output_cov:
fvar = (
tf.matrix_diag(
tf.tile(
(eKff - tf.trace(Li_eKuffu_Lit))[:, None], [1, num_func]))
+ tf.matrix_diag(tf.einsum("nij,dji->nd", Li_eKuffu_Lit, cov)) +
# tf.matrix_diag(tf.trace(tf.matmul(Li_eKuffu_Lit, cov))) +
tf.einsum("ig,nij,jh->ngh", q_mu, Li_eKuffu_Lit, q_mu) -
# tf.matmul(q_mu, tf.matmul(Li_eKuffu_Lit, q_mu), transpose_a=True) -
fmean[:, :, None] * fmean[:, None, :] + e_related_to_mean)
else:
fvar = ((eKff - tf.trace(Li_eKuffu_Lit))[:, None] +
tf.einsum("nij,dji->nd", Li_eKuffu_Lit, cov) +
tf.einsum("ig,nij,jg->ng", q_mu, Li_eKuffu_Lit, q_mu) -
fmean**2 + tf.matrix_diag_part(e_related_to_mean))
return fmean, fvar, cross_cov_terms
import tensorflow as tf
x = tf.compat.v1.placeholder(tf.int32, shape=[3], name='x')
y = tf.compat.v1.placeholder(tf.int32, shape=[3], name='y')
sum_x = tf.reduce_sum(x, name='sum_x')
prod_y = tf.reduce_prod(y, name='prod_y')
final_div = tf.div(sum_x, prod_y, name='final_div')
final_mean = tf.reduce_mean([sum_x, prod_y], name='final_mean')
sess = tf.Session()
print('sum(x): ', sess.run(sum_x, feed_dict = {x: [100,200,300]}))
print('prod(y): ', sess.run(prod_y, feed_dict = {y: [1,2,3]}))
writer = tf.compat.v1.summary.FileWriter('m3_example2', sess.graph)
writer.close()
sess.close()
def prod(x, axis=None, keepdims=False):
'''Multiply the values in a tensor, alongside the specified axis.
'''
return tf.reduce_prod(x, reduction_indices=axis, keep_dims=keepdims)
def main(train=True, timeString=None):
#Define Dataset Operations
trainDataSet, valdDataSet, testDataSet = GetDataSets()
imageShape = [None, height, width, n_channels]
maskShape = [None, height, width]
with tf.variable_scope('InputPlaceholders'):
imagesPL = tf.placeholder(dtype=tf.float32,
shape=imageShape,
name='imagesPL')
labelsPL = tf.placeholder(dtype=tf.float32,
shape=maskShape,
name='masksPL')
binaryLabels = tf.cast(labelsPL > 0, tf.int32)
valdImages, valdMasks = valdDataSet.NextBatch(
None) # Of shape (37, 96, 96, -1)
valdImages = np.swapaxes(valdImages, 0, 3)
valdMasks = np.swapaxes(valdMasks, 0, 3)
testImages, testMasks = testDataSet.NextBatch(None)
testFileNames = testDataSet.filenames
testImages = np.swapaxes(testImages, 0, 3)
testMasks = np.swapaxes(testMasks, 0, 3)
# Define Learning Parameters
globalStep = tf.Variable(0, trainable=False)
initLearningRate = 0.0001
learningRate = tf.train.exponential_decay(initLearningRate,
globalStep,
10000,
0.9,
staircase=True)
learningRateSummary = tf.summary.scalar('learningRate', learningRate)
#Build Model
outputLayer = CNN(imagesPL)
binaryOutputLayer = tf.argmax(outputLayer, axis=3, output_type=tf.int32)
inputSummary = tf.summary.merge([
tf.summary.image('Images', imagesPL, max_outputs=1),
tf.summary.image('TrueMasks',
tf.expand_dims(tf.cast(binaryLabels, tf.float32), -1),
max_outputs=1),
tf.summary.image('PredictedMasks',
tf.expand_dims(tf.cast(binaryOutputLayer, tf.float32),
-1),
max_outputs=1)
])
#Define Training Operation
with tf.variable_scope('LossOperations'):
lossOp = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=binaryLabels,
logits=outputLayer))
trainOp = tf.train.AdamOptimizer(learningRate).minimize(
lossOp, global_step=globalStep)
lossPL = tf.placeholder(dtype=tf.float32, shape=())
entropySummary = tf.summary.scalar('CrossEntropy', lossPL)
#Define Dice Coefficient
diceOp = Dice(binaryLabels, binaryOutputLayer)
evalDiceOp = PatientDice(binaryLabels, binaryOutputLayer)
dicePL = tf.placeholder(dtype=tf.float32, shape=())
diceSummary = tf.summary.scalar('DiceCoeff', dicePL)
with tf.variable_scope('Accuracy'):
accuracyOp = tf.reduce_sum(
tf.cast(tf.equal(binaryOutputLayer, binaryLabels),
tf.float32)) / tf.cast(
tf.reduce_prod(tf.shape(binaryLabels)), tf.float32)
with tf.variable_scope('TruePositive'):
numer = tf.reduce_sum(
tf.cast(binaryLabels, tf.float32) *
tf.cast(tf.equal(binaryOutputLayer, binaryLabels), tf.float32))
denom = tf.reduce_sum(tf.cast(binaryLabels, tf.float32))
truePositiveOp = numer / denom
with tf.variable_scope('TrueNegative'):
trueNegativeOp = tf.reduce_sum(
tf.cast(1 - binaryLabels, tf.float32) *
tf.cast(tf.equal(binaryOutputLayer, binaryLabels), tf.float32)
) / tf.reduce_sum(tf.cast(1 - binaryLabels, tf.float32))
#Initialize weights, sessions
if not train and timeString is None:
raise ValueError(
'Specified train=False but timeString was not provided. Specify a timeString to load a checkpoint from.'
)
elif train:
timeString = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M")
modelPath = '../checkpoints/{}/'.format(timeString)
summaryPath = '../summaries/{}/'.format(timeString)
if not os.path.exists(modelPath):
os.makedirs(modelPath)
if not os.path.exists(summaryPath):
os.makedirs(summaryPath)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
#Start Queue Runner for Data Input
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
#Define Summary Writer
mergedSummaries = tf.summary.merge(
[learningRateSummary, entropySummary, diceSummary])
trainWriter = tf.summary.FileWriter(summaryPath + 'train/', sess.graph)
evalWriter = tf.summary.FileWriter(summaryPath + 'eval/')
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
accumulatedSteps = 0
bestDice = 0
if train:
for i in range(cappedIterations):
trainImages, trainMasks = trainDataSet.NextBatch(sess)
feed_dict = {imagesPL: trainImages, labelsPL: trainMasks}
if i % batchStepsBetweenSummaries == 0:
_, trainingLoss, diceCoeff, imageSummaries = sess.run(
[trainOp, lossOp, diceOp, inputSummary],
feed_dict=feed_dict)
summaries = sess.run(mergedSummaries,
feed_dict={
lossPL: trainingLoss,
dicePL: diceCoeff
})
trainWriter.add_summary(summaries, i)
trainWriter.add_summary(imageSummaries, i)
valdLoss, valdDice, valdImageSummaries = getPerformanceOnImages(
sess, valdImages, valdMasks, imagesPL, labelsPL,
lossOp, evalDiceOp, inputSummary)
summaries = sess.run(mergedSummaries,
feed_dict={
lossPL: valdLoss,
dicePL: valdDice
})
evalWriter.add_summary(summaries, i)
evalWriter.add_summary(valdImageSummaries, i)
print(
'Iteration {}, Training: [Loss = {}, Dice = {}], Validation: [Loss = {}, Dice = {}]'
.format(i, trainingLoss, diceCoeff, valdLoss,
valdDice))
if valdDice > bestDice:
bestDice = valdDice
accumulatedSteps = 0
saver.save(sess, modelPath)
print('Model saved to path: {}'.format(modelPath))
else:
accumulatedSteps += batchStepsBetweenSummaries
if accumulatedSteps > stepsBeforeStoppingCriteria:
print(
'Reached early stopping criteria with validation dice {}'
.format(bestDice))
break
else:
sess.run([trainOp], feed_dict=feed_dict)
saver.restore(sess, modelPath)
print('Model restored from path: {}'.format(modelPath))
lower, mid, upper = getBootstrapPerformanceOnImages(
sess, testImages, testMasks, imagesPL, labelsPL, lossOp)
print('Test: Loss = {}, ({}, {})'.format(mid, lower, upper))
lower, mid, upper = getBootstrapPerformanceOnImages(
sess, testImages, testMasks, imagesPL, labelsPL, evalDiceOp)
print('Test: DICE = {}, ({}, {})'.format(mid, lower, upper))
lower, mid, upper = getBootstrapPerformanceOnImages(
sess, testImages, testMasks, imagesPL, labelsPL, accuracyOp)
print('Test: Accuracy = {}, ({}, {})'.format(mid, lower, upper))
lower, mid, upper = getBootstrapPerformanceOnImages(
sess, testImages, testMasks, imagesPL, labelsPL, truePositiveOp)
print('Test: TruePositive = {}, ({}, {})'.format(mid, lower, upper))
lower, mid, upper = getBootstrapPerformanceOnImages(
sess, testImages, testMasks, imagesPL, labelsPL, trueNegativeOp)
print('Test: TrueNegative = {}, ({}, {})'.format(mid, lower, upper))
df = pd.DataFrame(
data={
'Volume':
testFileNames,
'Dice':
getPatientLosses(sess, testImages, testMasks, imagesPL,
labelsPL, evalDiceOp),
'Accuracy':
getPatientLosses(sess, testImages, testMasks, imagesPL,
labelsPL, accuracyOp),
'Sensitivity':
getPatientLosses(sess, testImages, testMasks, imagesPL,
labelsPL, truePositiveOp),
'Specificty':
getPatientLosses(sess, testImages, testMasks, imagesPL,
labelsPL, trueNegativeOp)
})
df.to_csv('PatientMetrics.csv', index=False)
coord.request_stop()
coord.join(threads)
def main(unused_argv):
(image, product_image, body_segment, prod_segment, skin_segment, pose_map,
image_id) = build_input()
# Build model and loss function
model = create_model(product_image, body_segment, skin_segment, pose_map,
prod_segment, image)
# Summaries.
with tf.name_scope("encode_images"):
display_fetches = {
"paths":
image_id,
"image":
tf.map_fn(tf.image.encode_png,
deprocess_image(image),
dtype=tf.string,
name="image_pngs"),
"product_image":
tf.map_fn(tf.image.encode_png,
deprocess_image(product_image),
dtype=tf.string,
name="prod_image_pngs"),
"body_segment":
tf.map_fn(tf.image.encode_png,
deprocess_image(body_segment, True),
dtype=tf.string,
name="body_segment_pngs"),
"skin_segment":
tf.map_fn(tf.image.encode_png,
deprocess_image(skin_segment),
dtype=tf.string,
name="skin_segment_pngs"),
"prod_segment":
tf.map_fn(tf.image.encode_png,
deprocess_image(prod_segment, True),
dtype=tf.string,
name="prod_segment_pngs"),
"image_outputs":
tf.map_fn(tf.image.encode_png,
deprocess_image(model.image_outputs),
dtype=tf.string,
name="image_output_pngs"),
"mask_outputs":
tf.map_fn(tf.image.encode_png,
deprocess_image(model.mask_outputs),
dtype=tf.string,
name="mask_output_pngs"),
}
test_fetches = {
"image_outputs":
tf.map_fn(tf.image.encode_png,
deprocess_image(model.image_outputs),
dtype=tf.string,
name="image_output_pngs"),
"paths":
image_id,
}
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_mask_L1", model.gen_loss_mask_L1)
tf.summary.scalar("generator_loss_content_L1", model.gen_loss_content_L1)
tf.summary.scalar("perceptual_loss", model.perceptual_loss)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum(
[tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=100)
sv = tf.train.Supervisor(logdir=FLAGS.output_dir,
save_summaries_secs=0,
saver=None)
with sv.managed_session() as sess:
tf.logging.info("parameter_count = %d" % sess.run(parameter_count))
if FLAGS.checkpoint != "":
tf.logging.info("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint)
if checkpoint == None:
checkpoint = FLAGS.checkpoint
saver.restore(sess, checkpoint)
if FLAGS.mode == "test":
# testing
# at most, process the test data once
tf.logging.info("test!")
with open(os.path.join(FLAGS.output_dir, "options.json"),
"a") as f:
f.write(json.dumps(vars(FLAGS), sort_keys=True, indent=4))
start = time.time()
max_steps = FLAGS.number_of_steps
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0
or step == max_steps - 1)
results = sess.run(test_fetches)
image_dir = os.path.join(FLAGS.output_dir, "images")
if not os.path.exists(image_dir):
os.makedirs(image_dir)
for i, in_path in enumerate(results["paths"]):
name, _ = os.path.splitext(
os.path.basename(in_path.decode("utf8")))
filename = name + ".png"
out_path = os.path.join(image_dir, filename)
contents = results["image_outputs"][i]
with open(out_path, "wb") as f:
f.write(contents)
else:
# training
with open(os.path.join(FLAGS.output_dir, "options.json"),
"a") as f:
f.write(json.dumps(vars(FLAGS), sort_keys=True, indent=4))
start = time.time()
max_steps = FLAGS.number_of_steps
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0
or step == max_steps - 1)
options = None
run_metadata = None
if should(FLAGS.trace_freq):
options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(FLAGS.progress_freq):
fetches["gen_loss_content_L1"] = model.gen_loss_content_L1
fetches["gen_loss_mask_L1"] = model.gen_loss_mask_L1
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["perceptual_loss"] = model.perceptual_loss
if should(FLAGS.summary_freq):
fetches["summary"] = sv.summary_op
if should(FLAGS.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches,
options=options,
run_metadata=run_metadata)
if should(FLAGS.summary_freq):
tf.logging.info("recording summary")
sv.summary_writer.add_summary(results["summary"],
results["global_step"])
if should(FLAGS.display_freq):
tf.logging.info("saving display images")
filesets = save_images(results["display"],
image_dict=[
"body_segment", "skin_segment",
"prod_segment", "mask_outputs",
"product_image", "image",
"image_outputs"
],
output_dir=FLAGS.output_dir,
step=results["global_step"])
append_index(filesets,
image_dict=[
"body_segment", "skin_segment",
"prod_segment", "mask_outputs",
"product_image", "image", "image_outputs"
],
output_dir=FLAGS.output_dir,
step=True)
if should(FLAGS.trace_freq):
tf.logging.info("recording trace")
sv.summary_writer.add_run_metadata(
run_metadata, "step_%d" % results["global_step"])
if should(FLAGS.progress_freq):
# global_step will have the correct step count if we resume from a
# checkpoint
train_epoch = math.ceil(results["global_step"] /
FLAGS.number_of_samples)
rate = (step + 1) * FLAGS.batch_size / (time.time() -
start)
tf.logging.info(
"progress epoch %d step %d image/sec %0.1f" %
(train_epoch, results["global_step"], rate))
tf.logging.info("gen_loss_GAN: %f" %
results["gen_loss_GAN"])
tf.logging.info("gen_loss_mask_L1: %f" %
results["gen_loss_mask_L1"])
tf.logging.info("gen_loss_content_L1: %f" %
results["gen_loss_content_L1"])
tf.logging.info("perceptual_loss: %f" %
results["perceptual_loss"])
if should(FLAGS.save_freq):
tf.logging.info("saving model")
saver.save(sess,
os.path.join(FLAGS.output_dir, "model"),
global_step=sv.global_step)
if sv.should_stop():
break
def initialize_graph(self): # , action, step, pre_acc, search_space_size):
try:
# creating graph
# self.graph = tf.Graph().as_default()
tf.reset_default_graph()
print('Building graph...')
# max_depth = 18
# image_size = 228
# channels = 3
# batch_size = 32
# num_classes = 10
# learning_rate = 0.01
self.input_dimensions = self.input_dimensions.split('x')
# if('mnist' in self.dataset_name):
# self.inputs = tf.placeholder(tf.uint8, [None, int(self.input_dimensions[0]), int(self.input_dimensions[1]), int(self.input_dimensions[2])], name="inputs")
# elif 'cifar' in self.dataset_name:
if self.data_format == 'channels_last':
self.inputs = tf.placeholder(tf.float32,
shape=[
None,
int(self.input_dimensions[0]),
int(self.input_dimensions[1]),
int(self.input_dimensions[2])
],
name="inputs")
else:
self.inputs = tf.placeholder(tf.float32,
shape=[
None,
int(self.input_dimensions[2]),
int(self.input_dimensions[0]),
int(self.input_dimensions[1])
],
name="inputs")
self.labels = tf.placeholder(tf.int32,
shape=[None, self.num_classes],
name='label')
self.depth = tf.placeholder(tf.int32, shape=[], name='depth')
self.dropout_rate = tf.placeholder(tf.float32,
shape=[],
name="dropout")
self.learning_rate = tf.placeholder(tf.float32,
shape=[],
name='learning_rate')
self.momentum = tf.placeholder(tf.float32,
shape=[],
name='momentum')
# self.nesterov = tf.placeholder(tf.bool, shape=(), name='nesterov')
# labels_onehot = tf.one_hot(self.labels, 10)
# child network
self.model = ChildNetwork(self.inputs,
self.depth,
self.dropout_rate,
self.num_classes,
max_depth=self.max_depth,
initial_filters=self.initial_filters,
data_format=self.data_format)
logits, self.probs = self.model.stochastic_depth_conv2d(
mode='train')
self.loss, self.accuracy = self.model.classification_loss(
logits=logits, label=self.labels)
_, self.validation_accuracy = self.model.classification_loss(
logits=logits, label=self.labels)
# print([t.name for op in self.graph.get_operations() for t in op.values()])
# print([t for op in self.graph.get_operations() for t in op.values()])
self.global_step = tf.Variable(0, trainable=False)
self.optimizer = self.activation_fn['4'](
learning_rate=self.learning_rate,
momentum=self.momentum,
use_nesterov=True)
self.var_list = tf.trainable_variables()
self.train_op = self.optimizer.minimize(
self.loss, global_step=self.global_step)
# self.all_trainable_vars = [np.product(list(map(int, v.shape))) * v.dtype.size for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)]
self.all_trainable_vars = tf.reduce_sum(
[tf.reduce_prod(v.shape) for v in tf.trainable_variables()])
# np.sum([np.product([xi.value for xi in x.get_shape()]) for x in tf.all_variables()])
# if ('mnist' in self.dataset_name):
# self.train_batch_x, self.train_batch_y, self.test_batch_x, self.test_batch_y = self.dataset[0], self.dataset[1], self.dataset[2], self.dataset[3]
# elif ('cifar' in self.dataset_name):
self.train_batch_x, self.train_batch_y, self.test_batch_x, self.test_batch_y = self.dataset
self.num_steps = int(
math.ceil(len(self.train_batch_x) / self.train_batch_size))
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
tf.summary.scalar("network_trainable_variables",
self.all_trainable_vars)
tf.summary.scalar("network_depth", self.depth)
tf.summary.scalar("network_loss", self.loss)
tf.summary.scalar('network_training_accuracy', self.accuracy)
tf.summary.scalar('network_validation_accuracy',
self.validation_accuracy)
self.summaries_op = tf.summary.merge_all()
filename = self.log_dir + '/network/tb_logs/'
self.summary_writer = tf.summary.FileWriter(filename,
graph=self.sess.graph)
# vars = tf.trainable_variables()
# print(vars) # some infos about variables...
# vars_vals = self.sess.run(vars)
# for var, val in zip(vars, vars_vals):
# print("var: {}, value: {}".format(var.name, val))
self.saver = tf.train.Saver(max_to_keep=1)
ckpt = tf.train.latest_checkpoint(self.model_dir +
'network/model.chkt')
if ckpt and tf.train.checkpoint_exists(self.model_dir):
self.saver.restore(self.sess, ckpt)
logging(
self.model_dir + 'network/model.chkt' +
" model loaded successfully", self.logger, 'info')
except Exception as e:
logging("Get reward failed - " + str(e), self.logger, 'error')
def build_search_images(self):
"""Crop search images from the input image based on the last target position
1. The input image is scaled such that the area of target&context takes up to (scale_factor * z_image_size) ^ 2
2. Crop an image patch as large as x_image_size centered at the target center.
3. If the cropped image region is beyond the boundary of the input image, mean values are padded.
"""
config = self.config
size_z = config.z_image_size # 127
size_x = config.x_image_size # 255
context_amount = 0.5
num_scales = config.num_scales # 3
scales = np.arange(num_scales) - get_center(num_scales) # [-1,0,1]
assert np.sum(scales) == 0, 'scales should be symmetric'
search_factors = [
config.scale_step**x for x in scales
] # scale_step=1.0375, [0.9638554216867469, 1.0, 1.0375]
frame_sz = tf.shape(self.image)
target_yx = self.target_bbox_feed[0:2] #y,x
target_size = self.target_bbox_feed[2:4] # height, width
avg_chan = tf.reduce_mean(self.image, axis=(0, 1), name='avg_chan')
# Compute base values
base_z_size = target_size
base_z_context_size = base_z_size + context_amount * tf.reduce_sum(
base_z_size)
base_s_z = tf.sqrt(
tf.reduce_prod(base_z_context_size)) # Canonical size
base_scale_z = tf.div(tf.to_float(size_z), base_s_z)
d_search = (size_x - size_z) / 2.0
base_pad = tf.div(d_search, base_scale_z)
base_s_x = base_s_z + 2 * base_pad
base_scale_x = tf.div(tf.to_float(size_x), base_s_x)
boxes = []
for factor in search_factors:
s_x = factor * base_s_x
frame_sz_1 = tf.to_float(frame_sz[0:2] - 1)
topleft = tf.div(target_yx - get_center(s_x), frame_sz_1)
bottomright = tf.div(target_yx + get_center(s_x), frame_sz_1)
box = tf.concat([topleft, bottomright], axis=0)
boxes.append(box)
boxes = tf.stack(boxes)
scale_xs = []
for factor in search_factors:
scale_x = base_scale_x / factor
scale_xs.append(scale_x)
self.scale_xs = tf.stack(scale_xs)
# Note we use different padding values for each image
# while the original implementation uses only the average value
# of the first image for all images.
image_minus_avg = tf.expand_dims(self.image - avg_chan, 0)
image_cropped = tf.image.crop_and_resize(image_minus_avg,
boxes,
box_ind=tf.zeros((num_scales),
tf.int32),
crop_size=[size_x, size_x])
self.search_images = image_cropped + avg_chan
def _build_policy_head(self, input_state):
self.adv_actor_ph = tf.placeholder("float", [self.batch_size], name='advantage')
with tf.variable_scope(self.name+'/lstm_decoder') as vs:
self.action_outputs = tf.placeholder(tf.float32, [self.batch_size, None, self.num_actions+1], name='action_outputs')
self.action_inputs = tf.placeholder(tf.float32, [self.batch_size, None, self.num_actions+1], name='action_inputs')
self.decoder_seq_lengths = tf.placeholder(tf.int32, [self.batch_size], name='decoder_seq_lengths')
self.allowed_actions = tf.placeholder(tf.float32, [self.batch_size, None, self.num_actions+1], name='allowed_actions')
self.use_fixed_action = tf.placeholder(tf.bool, name='use_fixed_action')
self.temperature = tf.placeholder(tf.float32, name='temperature')
self.decoder_hidden_state_size = input_state.get_shape().as_list()[-1]
self.decoder_lstm_cell = CustomBasicLSTMCell(self.decoder_hidden_state_size, forget_bias=1.0)
self.decoder_initial_state = tf.placeholder(tf.float32, [self.batch_size, 2*self.decoder_hidden_state_size], name='decoder_initial_state')
self.network_state = tf.concat(axis=1, values=[
tf.zeros_like(input_state), input_state
# input_state, tf.zeros_like(input_state)
])
self.W_actions = tf.get_variable('W_actions', shape=[self.decoder_hidden_state_size, self.num_actions+1], dtype='float32', initializer=tf.contrib.layers.xavier_initializer())
self.b_actions = tf.get_variable('b_actions', shape=[self.num_actions+1], dtype='float32', initializer=tf.zeros_initializer())
self.decoder_state, self.logits, self.actions = decoder(
self.action_inputs,
self.network_state,
self.decoder_lstm_cell,
self.decoder_seq_lengths,
self.W_actions,
self.b_actions,
self.max_decoder_steps,
vs,
self.use_fixed_action,
self.action_outputs,
loop_function=loop_gumbel_softmax(self.temperature),
)
self.decoder_trainable_variables = [
v for v in tf.trainable_variables()
if v.name.startswith(vs.name)
]
print 'Decoder out: s,l,a=', self.decoder_state.get_shape(), self.logits.get_shape(), self.actions.get_shape()
#mask softmax by allowed actions
exp_logits = tf.exp(self.logits) * self.allowed_actions
Z = tf.expand_dims(tf.reduce_sum(exp_logits, 2), 2)
self.action_probs = exp_logits / Z
log_action_probs = self.logits - tf.log(Z)
sequence_probs = tf.reduce_prod(tf.reduce_sum(self.action_probs * self.action_outputs, 2), 1)
log_sequence_probs = tf.reduce_sum(tf.reduce_sum(log_action_probs * self.action_outputs, 2), 1)
# ∏a_i * ∑ log a_i
self.output_layer_entropy = - tf.reduce_sum(tf.stop_gradient(1 + log_sequence_probs) * log_sequence_probs)
self.entropy = - tf.reduce_sum(log_sequence_probs)
print 'sp, lsp:', sequence_probs.get_shape(), log_sequence_probs.get_shape()
self.actor_advantage_term = tf.reduce_sum(log_sequence_probs[:self.max_local_steps] * self.adv_actor_ph)
self.actor_entropy_term = self.beta * self.output_layer_entropy
self.actor_objective = - (
self.actor_advantage_term
+ self.actor_entropy_term
)
return self.actor_objective
def compress():
"""Compresses an image."""
# Load input image and add batch dimension.
# x = load_image(args.input)
# x = tf.expand_dims(x, 0)
# x.set_shape([1, None, None, 3])
x = tf.placeholder(tf.float32, [1, None, None, 3])
# Transform and compress the image, then remove batch dimension.
y = analysis_transform(x, args.num_filters)
entropy_bottleneck = tfc.EntropyBottleneck()
string = entropy_bottleneck.compress(y)
string = tf.squeeze(string, axis=0)
# Transform the quantized image back (if requested).
y_hat, likelihoods = entropy_bottleneck(y, training=False)
x_hat = synthesis_transform(y_hat, args.num_filters)
num_pixels = tf.to_float(tf.reduce_prod(tf.shape(x)[:-1]))
# Total number of bits divided by number of pixels.
eval_bpp = tf.reduce_sum(tf.log(likelihoods)) / (-np.log(2) * num_pixels)
# Mean squared error across pixels.
x_hat = tf.clip_by_value(x_hat, 0, 1)
x_hat = tf.round(x_hat * 255)
print(x_hat.shape)
mse = tf.reduce_sum(tf.squared_difference(x * 255, x_hat)) / num_pixels
with tf.Session() as sess:
# Load the latest model checkpoint and test images.
latest = tf.train.latest_checkpoint(checkpoint_dir=args.checkpoint_dir)
tf.train.Saver().restore(sess, save_path=latest)
dataset, img_names = _load_image()
for img, img_name in zip(dataset, img_names):
# Get the compressed string and the tensor shapes.
_string, x_shape, y_shape = sess.run(
[string, tf.shape(x), tf.shape(y)], feed_dict={x: [img]})
# Write a binary file with the shape information and the compressed string.
file_name = args.checkpoint_dir + '/results/' + img_name[:-4] + '.bin'
with open(file_name, "wb") as f:
# with open(args.output, "wb") as f:
f.write(np.array(x_shape[1:-1], dtype=np.uint16).tobytes())
f.write(np.array(y_shape[1:-1], dtype=np.uint16).tobytes())
f.write(_string)
# If requested, transform the quantized image back and measure performance.
if args.verbose:
# To print the results, the size of images must be a multiple of 16.
# eval_bpp, mse, num_pixels = sess.run([eval_bpp, mse, num_pixels], feed_dict={x: [img]})
_eval_bpp, _num_pixels = sess.run([eval_bpp, num_pixels],
feed_dict={x: [img]})
# The actual bits per pixel including overhead.
bpp = (8 + len(_string)) * 8 / _num_pixels
# print("Mean squared error: {:0.4}".format(mse))
print(
"Information content of this image in bpp: {:0.4}".format(
_eval_bpp))
print(
"Actual bits per pixel for this image: {:0.4}".format(bpp))
def fused_layer_norm(inputs,
center=True,
scale=True,
activation_fn=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
begin_norm_axis=1,
begin_params_axis=-1,
scope=None,
use_fused_batch_norm=False):
with tf.variable_scope(scope, 'LayerNorm', [inputs], reuse=reuse) as sc:
inputs = ops.convert_to_tensor(inputs)
inputs_shape = inputs.shape
inputs_rank = inputs_shape.ndims
if inputs_rank is None:
raise ValueError('Inputs %s has undefined rank.' % inputs.name)
dtype = inputs.dtype.base_dtype
if begin_norm_axis < 0:
begin_norm_axis = inputs_rank + begin_norm_axis
if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank:
raise ValueError('begin_params_axis (%d) and begin_norm_axis (%d) '
'must be < rank(inputs) (%d)' %
(begin_params_axis, begin_norm_axis, inputs_rank))
params_shape = inputs_shape[begin_params_axis:]
if not params_shape.is_fully_defined():
raise ValueError(
'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' %
(inputs.name, begin_params_axis, inputs_shape))
# Allocate parameters for the beta and gamma of the normalization.
beta, gamma = None, None
if center:
beta_collections = utils.get_variable_collections(
variables_collections, 'beta')
beta = variables.model_variable(
'beta',
shape=params_shape,
dtype=dtype,
initializer=init_ops.zeros_initializer(),
collections=beta_collections,
trainable=trainable)
if scale:
gamma_collections = utils.get_variable_collections(
variables_collections, 'gamma')
gamma = variables.model_variable(
'gamma',
shape=params_shape,
dtype=dtype,
initializer=init_ops.ones_initializer(),
collections=gamma_collections,
trainable=trainable)
if use_fused_batch_norm:
# get static TensorShape if fully defined,
# otherwise retrieve shape tensor
norm_shape = inputs.shape[begin_norm_axis:]
if norm_shape.is_fully_defined():
bn_shape = [1, -1, 1, numpy.prod(norm_shape.as_list())]
else:
norm_shape = tf.shape(inputs)[begin_norm_axis:]
bn_shape = [1, -1, 1, tf.reduce_prod(norm_shape)]
if inputs.get_shape().is_fully_defined():
outputs_shape = inputs.get_shape()
else:
outputs_shape = tf.shape(inputs)
inputs = array_ops.reshape(inputs, bn_shape)
if inputs.get_shape().is_fully_defined():
# static inputs TensorShape fully defined after reshape.
ones = array_ops.ones(inputs.get_shape()[1],
dtype=dtypes.float32)
zeros = array_ops.zeros(inputs.get_shape()[1],
dtype=dtypes.float32)
else:
# static inputs TensorShape NOT fully defined after reshape.
# must use dynamic shape, which means these input tensors
# have to be created at runtime, which causes a slowdown.
scale_shape = tf.shape(inputs)[1]
ones = array_ops.ones(scale_shape, dtype=dtypes.float32)
zeros = array_ops.zeros(scale_shape, dtype=dtypes.float32)
outputs, mean, variance = nn.fused_batch_norm(inputs,
ones,
zeros,
epsilon=1e-4,
data_format="NCHW")
outputs = array_ops.reshape(outputs, outputs_shape)
if center and scale:
outputs = outputs * gamma + beta
elif center:
outputs = outputs + beta
elif scale:
outputs = outputs * gamma
else:
# Calculate the moments on the last axis (layer activations).
norm_axes = list(range(begin_norm_axis, inputs_rank))
mean, variance = nn.moments(inputs, norm_axes, keep_dims=True)
# Compute layer normalization using the batch_normalization function.
variance_epsilon = 1e-4
outputs = nn.batch_normalization(inputs,
mean,
variance,
offset=beta,
scale=gamma,
variance_epsilon=variance_epsilon)
outputs.set_shape(inputs_shape)
if activation_fn is not None:
outputs = activation_fn(outputs)
return utils.collect_named_outputs(outputs_collections, sc.name,
outputs)
def __init__(self, **kwargs):
super(NAM_Modified, self).__init__()
self.lstm_layer = kwargs['lstm_layer']
self.lstm_dim = self.lstm_layer[-1]
self.balance = kwargs['balance']
self.layer_type = kwargs['layer_type']
self.decay = True
self.starter_learning_rate = kwargs['learning_rate_kge'] or 1e-4
self.learning_rate_lm = kwargs['learning_rate_lm'] or 1e-3
self.hidden_units_1 = kwargs['hidden_units_1'] or 1
self.hidden_units_2 = kwargs['hidden_units_2'] or None
self.hidden_units_3 = kwargs['hidden_units_3'] or None
self.hidden_units_4 = kwargs['hidden_units_4'] or None
self.drop_out = kwargs['dropout']
self.final = kwargs['final']
self.embedding_matrix = kwargs['embedding_matrix'].astype(np.float32)
self.task = kwargs['task']
self.vocabulary_size = self.embedding_matrix.shape[0]
# self.embedding_matrix =
self.max_norm = 5.0
if self.final and self.layer_type == 'BiLSTM':
self.Input_dimension = self.lstm_layer[-1] * 2
elif self.layer_type == 'BiLSTM' and not self.final:
self.Input_dimension = self.lstm_layer[-1] * 4
else:
self.Input_dimension = self.lstm_layer[-1]
with tf.device('/cpu:0'):
self.embedding_mat = tf.get_variable(initializer=self.embedding_matrix, name="embedding_matrix",
trainable=True, dtype=tf.float32)
with tf.device('/gpu:0'):
self.tail = tf.placeholder(dtype=tf.int32, name='y_true')
self.head = tf.placeholder(dtype=tf.int32, name='head_in')
self.rel = tf.placeholder(dtype=tf.int32, name='tail_in')
self.y_true_1 = tf.placeholder(dtype=tf.float32, name="y_true_values")
self.sequence_in = tf.placeholder(dtype=tf.float32, shape=(None, None, 300), name="sequence_in")
self.embed_head = tf.nn.embedding_lookup(self.embedding_mat, self.head)
self.embed_tail = tf.nn.embedding_lookup(self.embedding_mat, self.tail)
self.embed_relation = tf.nn.embedding_lookup(self.embedding_mat, self.rel)
self.embed_total = tf.concat([self.embed_head, self.embed_relation, self.embed_tail], axis=1)
self.sequence_sent = tf.placeholder(dtype=tf.int32, shape=(None, None), name='sentence_in')
self.task = "INIT"
self.current_view = 1
self.lstm_fw = [tf.contrib.rnn.BasicLSTMCell(size) for size in self.lstm_layer]
self.lstm_bw = [tf.contrib.rnn.BasicLSTMCell(size) for size in self.lstm_layer]
self.drops_fw = [tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=self.drop_out) for lstm in
self.lstm_fw]
self.drops_bw = [tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=self.drop_out) for lstm in
self.lstm_bw]
self.cell_fw = tf.contrib.rnn.MultiRNNCell(self.drops_fw)
self.cell_bw = tf.contrib.rnn.MultiRNNCell(self.drops_bw)
self.weights = {
'l1': tf.get_variable("Wl1", shape=[self.Input_dimension, self.hidden_units_1], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l2': tf.get_variable("Wl2", shape=[self.hidden_units_1, self.hidden_units_2], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l3': tf.get_variable("Wl3", shape=[self.hidden_units_2, self.hidden_units_3], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l4': tf.get_variable("Wl4", shape=[self.hidden_units_3, self.hidden_units_4], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
}
self.biases = {
'l1': tf.get_variable("bl1", shape=[self.hidden_units_1], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l2': tf.get_variable("bl2", shape=[self.hidden_units_2], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l3': tf.get_variable("bl3", shape=[self.hidden_units_3], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
'l4': tf.get_variable("bl4", shape=[self.hidden_units_4], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(), trainable=True),
}
if self.task == 'KGE' or self.task == 'INIT':
with tf.device('/gpu:0'):
if self.layer_type == 'BiLSTM':
self.lstm_output_1, self.final_state_1 = self.LSTM_Layer(self.sequence_in, time_major=False)
self.lstm_output_1 = tf.concat(self.lstm_output_1, 2)
self.final_state_fw, self.final_state_bw = self.final_state_1
self.final_state_c_fw, self.final_state_c_bw = self.final_state_fw[-1].c, self.final_state_bw[-1].c
self.final_state_c = tf.concat([self.final_state_c_fw, self.final_state_c_bw], axis=1)
logging.info("state 1 & 2 : {}{}".format(self.final_state_c_fw, self.final_state_c_bw))
if self.layer_type == 'LSTM':
self.lstm_output_1, self.final_state_1 = self.f_LSTM_Layer(self.sequence_in, time_major=False)
self.final_state_c, self.final_state_h = self.final_state_1[-1].c, self.final_state_1[-1].h
self.z0 = self.final_state_c
self.l1 = tf.matmul(self.z0, self.weights['l1']) + self.biases['l1']
self.z1 = tf.nn.relu(self.l1)
# self.z1 = tf.nn.dropout(self.z1, 0.1)
self.l2 = tf.matmul(self.z1, self.weights['l2']) + self.biases['l2']
self.z2 = tf.nn.relu(self.l2)
self.l3 = tf.matmul(self.z2, self.weights['l3']) + self.biases['l3']
self.z3 = tf.nn.relu(self.l3)
self.z3 = tf.nn.dropout(self.z3, 0.2)
self.l4 = tf.matmul(self.z3, self.weights['l4']) + self.biases['l4']
self.output = self.l4
self.output = tf.squeeze(self.output)
self.output_2 = tf.nn.sigmoid(self.output)
self.beta = 0.01
self.regularizer = tf.nn.l2_loss(self.weights['l3']) \
+ tf.nn.l2_loss(self.weights['l2']) + \
tf.nn.l2_loss(self.weights['l1']) + \
tf.nn.l2_loss(self.weights['l4'])
self.regularizer = self.beta * self.regularizer
self.cost = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(self.y_true_1, self.output, pos_weight=self.balance) +
self.regularizer)
if self.task == 'LM' or self.task == 'INIT':
with tf.device('/gpu:1'):
if self.layer_type == 'BiLSTM':
self.sentence_out = self.sequence_sent[:, 1:-1]
self.sentence_embed = tf.nn.embedding_lookup(self.embedding_mat, self.sequence_sent)
self.seq_len = tf.reduce_sum(tf.sign(self.sequence_sent), 1)
self.birnn_outputs, self.final_sentence = self.LSTM_Layer(input_=self.sentence_embed,
time_major=False,
sequence_length=self.seq_len)
self.fw_outputs = self.birnn_outputs[0][:, :-2, :]
self.bw_outputs = self.birnn_outputs[1][:, 2:, :]
self.final_output = self.fw_outputs[:, -1, :]
if self.layer_type == 'LSTM':
self.sentence_in = self.sequence_sent[:, :-1]
self.sentence_out = self.sequence_sent[:, 1:]
self.sentence_embed = tf.nn.embedding_lookup(self.embedding_mat, self.sentence_in)
self.seq_len = tf.reduce_sum(tf.sign(self.sentence_in), 1)
self.rnn_outputs, self.final_sentence = self.f_LSTM_Layer(input_=self.sentence_embed, time_major=False, sequence_length=self.seq_len)
self.fw_outputs = self.rnn_outputs
self.final_output = self.fw_outputs[:, -1, :]
with tf.device('/gpu:2'):
self.a = self.lstm_dim
self.weights = tf.get_variable(dtype=tf.float32, shape=(self.a, self.vocabulary_size),
name="softmax_weights")
self.bias = tf.get_variable(dtype=tf.float32, shape=self.vocabulary_size, name='softmax_bias')
self.logits = tf.matmul(self.fw_outputs, self.weights) + self.bias
self.next_output = tf.nn.softmax(tf.matmul(self.final_output, self.weights) + self.bias)
print(self.next_output.shape)
self.pred_logits = tf.nn.softmax(self.logits)
if self.layer_type == 'BiLSTM':
self.loss = tf.contrib.seq2seq.sequence_loss(logits=self.logits,
targets=self.sentence_out,
weights=tf.sequence_mask(
self.seq_len - 2,
tf.shape(self.sequence_sent)[
1] - 2,
dtype=tf.float32),
average_across_timesteps=False,
average_across_batch=False)
if self.layer_type == 'LSTM':
self.loss = tf.contrib.seq2seq.sequence_loss(logits=self.logits,
targets=self.sentence_out,
weights=tf.sequence_mask(
self.seq_len,
tf.shape(self.sequence_sent)[
1] - 1,
dtype=tf.float32),
average_across_timesteps=False,
average_across_batch=False)
self.output_loss = tf.reduce_mean(tf.reduce_mean(self.loss, axis=1), axis=0)
self.prediction_loss = tf.reduce_sum(tf.reduce_sum(self.loss, axis=1), axis=0)
if self.task == 'KGE' or self.task == 'INIT':
with tf.device('/gpu:0'):
self.global_step_2 = tf.Variable(0, trainable=False)
self.params_kge = tf.trainable_variables()
self.gradients_kge = tf.gradients(self.cost, self.params_kge)
self.clipped_gradients_kge, _ = tf.clip_by_global_norm(self.gradients_kge, self.max_norm)
self.learning_rate_2 = tf.train.exponential_decay(self.starter_learning_rate, self.global_step_2,
decay_steps=100000,
decay_rate=0.05, staircase=True)
self.optimizer_kge = tf.train.AdamOptimizer(self.learning_rate_2)
self.train_op = self.optimizer_kge.apply_gradients(zip(self.clipped_gradients_kge, self.params_kge),
global_step=self.global_step_2)
if self.task == 'LM' or self.task == "INIT":
with tf.device('/gpu:1'):
self.global_step_3 = tf.Variable(0, trainable=False)
self.params_lm = tf.trainable_variables()
self.gradients_lm = tf.gradients(self.output_loss, self.params_lm)
self.clipped_gradients_lm, _ = tf.clip_by_global_norm(self.gradients_lm, self.max_norm)
self.learning_rate_3 = tf.train.exponential_decay(self.learning_rate_lm, self.global_step_3,
decay_steps=100000,
decay_rate=0.5, staircase=True)
self.optimizer_lm = tf.train.AdamOptimizer(self.learning_rate_3)
self.optim_lm = self.optimizer_lm.apply_gradients(zip(self.clipped_gradients_lm, self.params_lm),
global_step=self.global_step_3)
self.saver = tf.train.Saver(tf.trainable_variables())
self.config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)
self.config.gpu_options.allow_growth = True
self.init = tf.global_variables_initializer()
self.session = tf.Session(config=self.config)
self.total_var_num = tf.reduce_sum([tf.reduce_prod(v.shape) for v in tf.trainable_variables()])
self.session.run(self.init)
def ms_ssim_calc(img1,
img2,
max_val=1.0,
power_factors=(0.0517, 0.3295, 0.3462, 0.2726)):
"""
Computes the MS-SSIM between img1 and img2.
This function assumes that `img1` and `img2` are image batches, i.e. the last
three dimensions are [height, width, channels].
Note: The true SSIM is only defined on grayscale. This function does not
perform any colorspace transform. (If input is already YUV, then it will
compute YUV SSIM average.)
Original paper: Wang, Zhou, Eero P. Simoncelli, and Alan C. Bovik. "Multiscale
structural similarity for image quality assessment." Signals, Systems and
Computers, 2004.
Arguments:
img1: First image batch.
img2: Second image batch. Must have the same rank as img1.
max_val: The dynamic range of the images (i.e., the difference between the
maximum the and minimum allowed values).
power_factors: Iterable of weights for each of the scales. The number of
scales used is the length of the list. Index 0 is the unscaled
resolution's weight and each increasing scale corresponds to the image
being downsampled by 2. Defaults to (0.0448, 0.2856, 0.3001, 0.2363,
0.1333), which are the values obtained in the original paper.
Returns:
A tensor containing an MS-SSIM value for each image in batch. The values
are in range [0, 1]. Returns a tensor with shape:
broadcast(img1.shape[:-3], img2.shape[:-3]).
"""
def _verify_compatible_image_shapes(img1, img2):
"""
Checks if two image tensors are compatible for applying SSIM or PSNR.
This function checks if two sets of images have ranks at least 3, and if the
last three dimensions match.
Args:
img1: Tensor containing the first image batch.
img2: Tensor containing the second image batch.
Returns:
A tuple containing: the first tensor shape, the second tensor shape, and a
list of control_flow_ops.Assert() ops implementing the checks.
Raises:
ValueError: When static shape check fails.
"""
shape1 = img1.get_shape().with_rank_at_least(3)
shape2 = img2.get_shape().with_rank_at_least(3)
shape1[-3:].assert_is_compatible_with(shape2[-3:])
if shape1.ndims is not None and shape2.ndims is not None:
for dim1, dim2 in zip(reversed(shape1[:-3]),
reversed(shape2[:-3])):
if not (dim1 == 1 or dim2 == 1
or dim1.is_compatible_with(dim2)):
raise ValueError(
'Two images are not compatible: %s and %s' %
(shape1, shape2))
# Now assign shape tensors.
shape1, shape2 = tf.shape_n([img1, img2])
# TODO(sjhwang): Check if shape1[:-3] and shape2[:-3] are broadcastable.
checks = []
checks.append(
tf.Assert(tf.greater_equal(tf.size(shape1), 3), [shape1, shape2],
summarize=10))
checks.append(
tf.Assert(tf.reduce_all(tf.equal(shape1[-3:], shape2[-3:])),
[shape1, shape2],
summarize=10))
return shape1, shape2, checks
def _ssim_per_channel(img1, img2, max_val=1.0):
"""
Computes SSIM index between img1 and img2 per color channel.
This function matches the standard SSIM implementation from:
Wang, Z., Bovik, A. C., Sheikh, H. R., & Simoncelli, E. P. (2004). Image
quality assessment: from error visibility to structural similarity. IEEE
transactions on image processing.
Details:
- 11x11 Gaussian filter of width 1.5 is used.
- k1 = 0.01, k2 = 0.03 as in the original paper.
Args:
img1: First image batch.
img2: Second image batch.
max_val: The dynamic range of the images (i.e., the difference between the
maximum the and minimum allowed values).
Returns:
A pair of tensors containing and channel-wise SSIM and contrast-structure
values. The shape is [..., channels].
"""
def _fspecial_gauss(size, sigma):
""" Function to mimic the 'fspecial' gaussian MATLAB function. """
size = tf.convert_to_tensor(size, 'int32')
sigma = tf.convert_to_tensor(sigma)
coords = tf.cast(tf.range(size), sigma.dtype)
coords -= tf.cast(size - 1, sigma.dtype) / 2.0
gauss = tf.square(coords)
gauss *= -0.5 / tf.square(sigma)
gauss = tf.reshape(gauss, shape=[1, -1]) + tf.reshape(
gauss, shape=[-1, 1])
gauss = tf.reshape(gauss, shape=[1, -1]) # For tf.nn.softmax().
gauss = tf.nn.softmax(gauss)
return tf.reshape(gauss, shape=[size, size, 1, 1])
def _ssim_helper(img1, img2, max_val, kernel, compensation=1.):
"""
Helper function for computing SSIM.
SSIM estimates covariances with weighted sums. The default parameters
use a biased estimate of the covariance:
Suppose `reducer` is a weighted sum, then the mean estimators are
mu_x = sum_i w_i x_i,
mu_y = sum_i w_i y_i,
where w_i's are the weighted-sum weights, and covariance estimator is
cov_{xy} = sum_i w_i (x_i - mu_x) (y_i - mu_y)
with assumption sum_i w_i = 1. This covariance estimator is biased, since
E[cov_{xy}] = (1 - sum_i w_i ^ 2) Cov(X, Y).
For SSIM measure with unbiased covariance estimators, pass as `compensation`
argument (1 - sum_i w_i ^ 2).
Arguments:
img1: First set of images.
img2: Second set of images.
reducer: Function that computes 'local' averages from set of images.
For non-covolutional version, this is usually tf.reduce_mean(img1, [1, 2]),
and for convolutional version, this is usually tf.nn.avg_pool or
tf.nn.conv2d with weighted-sum kernel.
max_val: The dynamic range (i.e., the difference between the maximum
possible allowed value and the minimum allowed value).
compensation: Compensation factor. See above.
Returns:
A pair containing the luminance measure, and the contrast-structure measure.
"""
def reducer(img1, kernel):
shape = tf.shape(img1)
img1 = tf.reshape(img1, shape=tf.concat([[-1], shape[-3:]], 0))
img2 = tf.nn.depthwise_conv2d(img1,
kernel,
strides=[1, 1, 1, 1],
padding='VALID')
return tf.reshape(
img2,
tf.concat([shape[:-3], tf.shape(img2)[1:]], 0))
c_one = (0.01 * max_val)**2
c_two = ((0.03 * max_val))**2 * compensation
# SSIM luminance measure is
# (2 * mu_x * mu_y + c_one) / (mu_x ** 2 + mu_y ** 2 + c_one).
mean0 = reducer(img1, kernel)
mean1 = reducer(img2, kernel)
num0 = mean0 * mean1 * 2.
den0 = tf.square(mean0) + tf.square(mean1)
luminance = (num0 + c_one) / (den0 + c_one)
# SSIM contrast-structure measure is
# (2 * cov_{xy} + c_two) / (cov_{xx} + cov_{yy} + c_two).
# Note that `reducer` is a weighted sum with weight w_k, \sum_i w_i = 1, then
# cov_{xy} = \sum_i w_i (x_i - \mu_x) (y_i - \mu_y)
# = \sum_i w_i x_i y_i - (\sum_i w_i x_i) (\sum_j w_j y_j).
num1 = reducer(img1 * img2, kernel) * 2.0
den1 = reducer(tf.square(img1) + tf.square(img2), kernel)
c_s = (num1 - num0 + c_two) / (den1 - den0 + c_two)
# SSIM score is the product of the luminance and contrast-structure measures.
return luminance, c_s
filter_size = tf.constant(9, dtype='int32') # changed from 11 to 9 due
filter_sigma = tf.constant(1.5, dtype=img1.dtype)
shape1, shape2 = tf.shape_n([img1, img2])
checks = [
tf.Assert(tf.reduce_all(
tf.greater_equal(shape1[-3:-1], filter_size)),
[shape1, filter_size],
summarize=8),
tf.Assert(tf.reduce_all(
tf.greater_equal(shape2[-3:-1], filter_size)),
[shape2, filter_size],
summarize=8)
]
# Enforce the check to run before computation.
with tf.control_dependencies(checks):
img1 = tf.identity(img1)
# TODO(sjhwang): Try to cache kernels and compensation factor.
kernel = _fspecial_gauss(filter_size, filter_sigma)
kernel = tf.tile(kernel, multiples=[1, 1, shape1[-1], 1])
# The correct compensation factor is `1.0 - tf.reduce_sum(tf.square(kernel))`,
# but to match MATLAB implementation of MS-SSIM, we use 1.0 instead.
compensation = 1.
# TODO(sjhwang): Try FFT.
# TODO(sjhwang): Gaussian kernel is separable in space. Consider applying
# 1-by-n and n-by-1 Gaussain filters instead of an n-by-n filter.
luminance, c_s = _ssim_helper(img1, img2, max_val, kernel,
compensation)
# Average over the second and the third from the last: height, width.
axes = tf.constant([-3, -2], dtype='int32')
ssim_val = tf.reduce_mean(luminance * c_s, axes)
c_s = tf.reduce_mean(c_s, axes)
return ssim_val, c_s
def do_pad(images, remainder):
padding = tf.expand_dims(remainder, -1)
padding = tf.pad(padding, [[1, 0], [1, 0]])
return [tf.pad(x, padding, mode='SYMMETRIC') for x in images]
# Shape checking.
shape1 = img1.get_shape().with_rank_at_least(3)
shape2 = img2.get_shape().with_rank_at_least(3)
shape1[-3:].merge_with(shape2[-3:])
with tf.name_scope(None, 'MS-SSIM', [img1, img2]):
shape1, shape2, checks = _verify_compatible_image_shapes(img1, img2)
with tf.control_dependencies(checks):
img1 = tf.identity(img1)
# Need to convert the images to float32. Scale max_val accordingly so that
# SSIM is computed correctly.
max_val = tf.cast(max_val, img1.dtype)
max_val = tf.image.convert_image_dtype(max_val, 'float32')
img1 = tf.image.convert_image_dtype(img1, 'float32')
img2 = tf.image.convert_image_dtype(img2, 'float32')
imgs = [img1, img2]
shapes = [shape1, shape2]
# img1 and img2 are assumed to be a (multi-dimensional) batch of
# 3-dimensional images (height, width, channels). `heads` contain the batch
# dimensions, and `tails` contain the image dimensions.
heads = [s[:-3] for s in shapes]
tails = [s[-3:] for s in shapes]
divisor = [1, 2, 2, 1]
divisor_tensor = tf.constant(divisor[1:], dtype='int32')
mc_s = []
for k in range(len(power_factors)):
with tf.name_scope(None, 'Scale%d' % k, imgs):
if k > 0:
# Avg pool takes rank 4 tensors. Flatten leading dimensions.
zipped = zip(imgs, tails)
flat_imgs = [
tf.reshape(x, tf.concat([[-1], t], 0)) for x, t in zipped
]
remainder = tails[0] % divisor_tensor
need_padding = tf.reduce_any(tf.not_equal(remainder, 0))
padded = tf.cond(need_padding,
lambda: do_pad(flat_imgs, remainder),
lambda: flat_imgs)
downscaled = [
tf.nn.avg_pool(x,
ksize=divisor,
strides=divisor,
padding='VALID') for x in padded
]
tails = [x[1:] for x in tf.shape_n(downscaled)]
zipper = zip(downscaled, heads, tails)
imgs = [
tf.reshape(x, tf.concat([h, t], 0)) for x, h, t in zipper
]
# Overwrite previous ssim value since we only need the last one.
ssim_per_channel, c_s = _ssim_per_channel(*imgs, max_val=max_val)
mc_s.append(tf.nn.relu(c_s))
# Remove the c_s score for the last scale. In the MS-SSIM calculation,
# we use the l(p) at the highest scale. l(p) * c_s(p) is ssim(p).
mc_s.pop() # Remove the c_s score for the last scale.
mcs_and_ssim = tf.stack(mc_s + [tf.nn.relu(ssim_per_channel)], axis=-1)
# Take weighted geometric mean across the scale axis.
ms_ssim = tf.reduce_prod(tf.pow(mcs_and_ssim, power_factors), [-1])
return tf.reduce_mean(ms_ssim, [-1]) # Avg over color channels.
def main():
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
if a.mode == "export":
# export the generator to a meta graph that can be imported later for standalone generation
input = tf.placeholder(tf.string, shape=[1])
input_data = tf.decode_base64(input[0])
input_image = tf.image.decode_png(input_data)
# remove alpha channel if present
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 4), lambda: input_image[:,:,:3], lambda: input_image)
# convert grayscale to RGB
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1), lambda: tf.image.grayscale_to_rgb(input_image), lambda: input_image)
input_image = tf.image.convert_image_dtype(input_image, dtype=tf.float32)
input_image.set_shape([CROP_SIZE, CROP_SIZE, 3])
batch_input = tf.expand_dims(input_image, axis=0)
with tf.variable_scope("generator"):
batch_output = deprocess(create_generator(preprocess(batch_input), 3))
output_image = tf.image.convert_image_dtype(batch_output, dtype=tf.uint8)[0]
if a.output_filetype == "png":
output_data = tf.image.encode_png(output_image)
elif a.output_filetype == "jpeg":
output_data = tf.image.encode_jpeg(output_image, quality=80)
else:
raise Exception("invalid filetype")
output = tf.convert_to_tensor([tf.encode_base64(output_data)])
key = tf.placeholder(tf.string, shape=[1])
inputs = {
"key": key.name,
"input": input.name
}
tf.add_to_collection("inputs", json.dumps(inputs))
outputs = {
"key": tf.identity(key).name,
"output": output.name,
}
tf.add_to_collection("outputs", json.dumps(outputs))
init_op = tf.global_variables_initializer()
restore_saver = tf.train.Saver()
export_saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
restore_saver.restore(sess, checkpoint)
print("exporting model")
export_saver.export_meta_graph(filename=os.path.join(a.output_dir, "export.meta"))
export_saver.save(sess, os.path.join(a.output_dir, "export"), write_meta_graph=False)
return
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputs, examples.targets)
# undo colorization splitting on images that we use for display/output
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths": examples.paths,
"inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"),
"targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"),
"outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"),
}
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
with tf.name_scope("predict_real_summary"):
tf.summary.image("predict_real", tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image("predict_fake", tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
# train
results = sess.run(fetches, options=options, run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"], step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch)
train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print("progress epoch %d step %d image/sec %0.1f remaining %dm" % (train_epoch, train_step, rate, remaining / 60))
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
if should(a.save_freq):
print("saving model")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
def __init__(self,
mixture_distribution,
components_distribution,
reparameterize=False,
validate_args=False,
allow_nan_stats=True,
name="MixtureSameFamily"):
"""Construct a `MixtureSameFamily` distribution.
Args:
mixture_distribution: `tfp.distributions.Categorical`-like instance.
Manages the probability of selecting components. The number of
categories must match the rightmost batch dimension of the
`components_distribution`. Must have either scalar `batch_shape` or
`batch_shape` matching `components_distribution.batch_shape[:-1]`.
components_distribution: `tfp.distributions.Distribution`-like instance.
Right-most batch dimension indexes components.
reparameterize: Python `bool`, default `False`. Whether to reparameterize
samples of the distribution using implicit reparameterization gradients
[(Figurnov et al., 2018)][1]. The gradients for the mixture logits are
equivalent to the ones described by [(Graves, 2016)][2]. The gradients
for the components parameters are also computed using implicit
reparameterization (as opposed to ancestral sampling), meaning that
all components are updated every step.
Only works when:
(1) components_distribution is fully reparameterized;
(2) components_distribution is either a scalar distribution or
fully factorized (tfd.Independent applied to a scalar distribution);
(3) batch shape has a known rank.
Experimental, may be slow and produce infs/NaNs.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
ValueError: `if not mixture_distribution.dtype.is_integer`.
ValueError: if mixture_distribution does not have scalar `event_shape`.
ValueError: if `mixture_distribution.batch_shape` and
`components_distribution.batch_shape[:-1]` are both fully defined and
the former is neither scalar nor equal to the latter.
ValueError: if `mixture_distribution` categories does not equal
`components_distribution` rightmost batch shape.
#### References
[1]: Michael Figurnov, Shakir Mohamed and Andriy Mnih. Implicit
reparameterization gradients. In _Neural Information Processing
Systems_, 2018. https://arxiv.org/abs/1805.08498
[2]: Alex Graves. Stochastic Backpropagation through Mixture Density
Distributions. _arXiv_, 2016. https://arxiv.org/abs/1607.05690
"""
parameters = dict(locals())
with tf.compat.v2.name_scope(name) as name:
self._mixture_distribution = mixture_distribution
self._components_distribution = components_distribution
self._runtime_assertions = []
s = components_distribution.event_shape_tensor()
self._event_ndims = tf.compat.dimension_value(s.shape[0])
if self._event_ndims is None:
self._event_ndims = tf.size(input=s)
self._event_size = tf.reduce_prod(input_tensor=s)
if not mixture_distribution.dtype.is_integer:
raise ValueError(
"`mixture_distribution.dtype` ({}) is not over integers".
format(mixture_distribution.dtype.name))
if (mixture_distribution.event_shape.ndims is not None
and mixture_distribution.event_shape.ndims != 0):
raise ValueError(
"`mixture_distribution` must have scalar `event_dim`s")
elif validate_args:
self._runtime_assertions += [
assert_util.assert_equal(
tf.size(
input=mixture_distribution.event_shape_tensor()),
0,
message=
"`mixture_distribution` must have scalar `event_dim`s"
),
]
mdbs = mixture_distribution.batch_shape
cdbs = components_distribution.batch_shape.with_rank_at_least(
1)[:-1]
if mdbs.is_fully_defined() and cdbs.is_fully_defined():
if mdbs.ndims != 0 and mdbs != cdbs:
raise ValueError(
"`mixture_distribution.batch_shape` (`{}`) is not "
"compatible with `components_distribution.batch_shape` "
"(`{}`)".format(mdbs.as_list(), cdbs.as_list()))
elif validate_args:
mdbs = mixture_distribution.batch_shape_tensor()
cdbs = components_distribution.batch_shape_tensor()[:-1]
self._runtime_assertions += [
assert_util.assert_equal(
distribution_utils.pick_vector(
mixture_distribution.is_scalar_batch(), cdbs,
mdbs),
cdbs,
message=
("`mixture_distribution.batch_shape` is not "
"compatible with `components_distribution.batch_shape`"
))
]
km = tf.compat.dimension_value(
mixture_distribution.logits.shape.with_rank_at_least(1)[-1])
kc = tf.compat.dimension_value(
components_distribution.batch_shape.with_rank_at_least(1)[-1])
if km is not None and kc is not None and km != kc:
raise ValueError(
"`mixture_distribution components` ({}) does not "
"equal `components_distribution.batch_shape[-1]` "
"({})".format(km, kc))
elif validate_args:
km = tf.shape(input=mixture_distribution.logits)[-1]
kc = components_distribution.batch_shape_tensor()[-1]
self._runtime_assertions += [
assert_util.assert_equal(
km,
kc,
message=(
"`mixture_distribution components` does not equal "
"`components_distribution.batch_shape[-1:]`")),
]
elif km is None:
km = tf.shape(input=mixture_distribution.logits)[-1]
self._num_components = km
self._reparameterize = reparameterize
if reparameterize:
# Note: tfd.Independent passes through the reparameterization type hence
# we do not need separate logic for Independent.
if (self._components_distribution.reparameterization_type !=
reparameterization.FULLY_REPARAMETERIZED):
raise ValueError("Cannot reparameterize a mixture of "
"non-reparameterized components.")
reparameterization_type = reparameterization.FULLY_REPARAMETERIZED
else:
reparameterization_type = reparameterization.NOT_REPARAMETERIZED
super(MixtureSameFamily, self).__init__(
dtype=self._components_distribution.dtype,
reparameterization_type=reparameterization_type,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=(
self._mixture_distribution._graph_parents # pylint: disable=protected-access
+ self._components_distribution._graph_parents), # pylint: disable=protected-access
name=name)
