def generalised_dice_loss(prediction,
ground_truth,
weight_map=None,
type_weight='Square'):
"""
Function to calculate the Generalised Dice Loss defined in
Sudre, C. et. al. (2017) Generalised Dice overlap as a deep learning
loss function for highly unbalanced segmentations. DLMIA 2017
:param prediction: the logits
:param ground_truth: the segmentation ground truth
:param weight_map:
:param type_weight: type of weighting allowed between labels (choice
between Square (square of inverse of volume),
Simple (inverse of volume) and Uniform (no weighting))
:return: the loss
"""
ground_truth = tf.to_int64(ground_truth)
n_voxels = ground_truth.shape[0].value
n_classes = prediction.shape[1].value
ids = tf.constant(np.arange(n_voxels), dtype=tf.int64)
ids = tf.stack([ids, ground_truth], axis=1)
one_hot = tf.SparseTensor(indices=ids,
values=tf.ones([n_voxels], dtype=tf.float32),
dense_shape=[n_voxels, n_classes])
if weight_map is not None:
weight_map_nclasses = tf.reshape(
tf.tile(weight_map, [n_classes]), prediction.get_shape())
ref_vol = tf.sparse_reduce_sum(
weight_map_nclasses * one_hot, reduction_axes=[0])
intersect = tf.sparse_reduce_sum(
weight_map_nclasses * one_hot * prediction, reduction_axes=[0])
seg_vol = tf.reduce_sum(
tf.multiply(weight_map_nclasses, prediction), 0)
else:
ref_vol = tf.sparse_reduce_sum(one_hot, reduction_axes=[0])
intersect = tf.sparse_reduce_sum(one_hot * prediction,
reduction_axes=[0])
seg_vol = tf.reduce_sum(prediction, 0)
if type_weight == 'Square':
weights = tf.reciprocal(tf.square(ref_vol))
elif type_weight == 'Simple':
weights = tf.reciprocal(ref_vol)
elif type_weight == 'Uniform':
weights = tf.ones_like(ref_vol)
else:
raise ValueError("The variable type_weight \"{}\""
"is not defined.".format(type_weight))
new_weights = tf.where(tf.is_inf(weights), tf.zeros_like(weights), weights)
weights = tf.where(tf.is_inf(weights), tf.ones_like(weights) *
tf.reduce_max(new_weights), weights)
generalised_dice_numerator = \
2 * tf.reduce_sum(tf.multiply(weights, intersect))
generalised_dice_denominator = \
tf.reduce_sum(tf.multiply(weights, seg_vol + ref_vol))
generalised_dice_score = \
generalised_dice_numerator / generalised_dice_denominator
return 1 - generalised_dice_score
def inverse_gamma_log_prob(self, val): conc = self.parameters['concentration'] rate = self.parameters['rate'] result = -(conc + 1) * tf.log(val) result -= rate * tf.reciprocal(val) result += -tf.lgamma(conc) + conc * tf.log(rate) return result
def normal_log_prob(self, val):
loc = self.parameters['loc']
scale = self.parameters['scale']
prec = tf.reciprocal(tf.square(scale))
result = prec * (-0.5 * tf.square(val) - 0.5 * tf.square(loc) +
val * loc)
result -= tf.log(scale) + 0.5 * tf.log(2 * np.pi)
return result
def mvn_diag_log_prob(self, val):
loc = self.parameters['loc']
scale_diag = self.parameters['scale_diag']
prec = tf.reciprocal(tf.square(scale_diag))
result = prec * (-0.5 * tf.square(val) - 0.5 * tf.square(loc) +
val * loc)
result -= tf.log(scale_diag) + 0.5 * tf.log(2 * np.pi)
return result
def integrandmat(inx, iny, th):
my2 = tf.constant(2.0,gl.myftype)
tfmu = tf.add(iny,tf.multiply(tff(theta=th,x=iny),gl.h))
tfsig = tf.multiply(tf.sqrt(gl.h),tfg(theta=th,x=iny))
tfc0 = tf.reciprocal(tf.multiply(tf.sqrt(tf.multiply(my2,tf.constant(np.pi,dtype=gl.myftype))),tfsig))
tfnumer = tf.negative(tf.square(tf.subtract(inx,tfmu)))
tfdenom = tf.multiply(my2,tf.square(tfsig))
tfprop = tf.multiply(tfc0,tf.exp(tf.divide(tfnumer,tfdenom)))
return tfprop
def cauchy_1d(sigma, truncated=5.0):
if sigma <= 0:
return tf.constant(0.0)
tail = int(sigma * truncated + 0.5)
k = [((float(x) / sigma) ** 2 + 1.0) for x in range(-tail, tail + 1)]
k = tf.reciprocal(k)
k = k / tf.reduce_sum(k)
return k
def _step(self, j):
j -= self.one
# note: we convert all the j to be positive before this calculation
# (even though we'll only use the values that are already positive),
# otherwise we can end up with nans in the gradient
rates = self.amplitude / (
self.tau_ref + self.tau_rc * tf.log1p(tf.reciprocal(
tf.maximum(j, self.epsilon))))
return tf.where(j > self.zero, rates, self.zeros)
def __call__(self, numerator, denominator):
"""
Acts as a normal tensorflow math function, performing save regularized division. Implemented as a class so that
it can follow the tf.div signature. Adds division loss (threshold penalty) to loss collections.
"""
step = tf.train.get_or_create_global_step()
div_thresh = self.div_thresh_fn(step)
mask = tf.cast(denominator > div_thresh, dtype=tf.float32)
div = tf.reciprocal(tf.abs(denominator) + 1e-10)
output = numerator * div * mask
P_theta = tf.maximum((div_thresh - denominator), 0.0) # equation 5 in paper
tf.add_to_collection('Threshold_penalties', P_theta)
return output
def _get_mixture_coef(self, output):
out_pi = tf.placeholder(dtype=tf.float64, shape=[None, self.KMIX], name="mixparam")
out_sigma = tf.placeholder(dtype=tf.float64, shape=[None, self.KMIX], name="mixparam")
out_mu = tf.placeholder(dtype=tf.float64, shape=[None, self.KMIX], name="mixparam")
out_pi, out_sigma, out_mu = tf.split(output, 3, 1)
max_pi = tf.reduce_max(out_pi, 1, keep_dims=True)
out_pi = tf.subtract(out_pi, max_pi)
out_pi = tf.exp(out_pi)
normalize_pi = tf.reciprocal(tf.reduce_sum(out_pi, 1, keep_dims=True))
out_pi = tf.multiply(normalize_pi, out_pi)
out_sigma = tf.exp(out_sigma)
tmp1 = tf.constant(0.0001, dtype=tf.float64)
out_sigma = tf.maximum(tmp1, out_sigma)
return out_pi, out_sigma, out_mu
def get_mixture_coef(output):
out_pi = tf.placeholder(dtype=tf.float32, shape=[None,KMIX], name="mixparam")
out_sigma = tf.placeholder(dtype=tf.float32, shape=[None,KMIX], name="mixparam")
out_mu = tf.placeholder(dtype=tf.float32, shape=[None,KMIX], name="mixparam")
out_pi, out_sigma, out_mu = tf.split(output, 3, 1)
max_pi = tf.reduce_max(out_pi, axis=1, keep_dims=True)
out_pi = tf.subtract(out_pi, max_pi)
out_pi = tf.exp(out_pi)
normalize_pi = tf.reciprocal(tf.reduce_sum(out_pi, 1, keep_dims=True))
out_pi = tf.multiply(normalize_pi, out_pi)
out_sigma = tf.exp(out_sigma)
return out_pi, out_sigma, out_mu
def _step(self, J):
J -= self.one
js = J / self.sigma
j_valid = js > -20
js_safe = tf.where(j_valid, js, self.zeros)
# softplus(js) = log(1 + e^js)
z = tf.nn.softplus(js_safe) * self.sigma
# as z->0
# z = s*log(1 + e^js) = s*e^js
# log(1 + 1/z) = log(1/z) = -log(s*e^js) = -js - log(s)
q = tf.where(j_valid,
tf.log1p(tf.reciprocal(z)),
-js - tf.log(self.sigma))
rates = self.amplitude / (self.tau_ref + self.tau_rc * q)
return rates
def rasterize_triangles(vertices, attributes, triangles, projection_matrices,
image_width, image_height, background_value):
"""Rasterizes the input scene and computes interpolated vertex attributes.
NOTE: the rasterizer does no triangle clipping. Triangles that lie outside the
viewing frustum (esp. behind the camera) may be drawn incorrectly.
Args:
vertices: 3-D float32 tensor with shape [batch_size, vertex_count, 3]. Each
triplet is an xyz position in model space.
attributes: 3-D float32 tensor with shape [batch_size, vertex_count,
attribute_count]. Each vertex attribute is interpolated
across the triangle using barycentric interpolation.
triangles: 2-D int32 tensor with shape [triangle_count, 3]. Each triplet
should contain vertex indices describing a triangle such that the
triangle's normal points toward the viewer if the forward order of the
triplet defines a clockwise winding of the vertices. Gradients with
respect to this tensor are not available.
projection_matrices: 3-D float tensor with shape [batch_size, 4, 4]
containing model-view-perspective projection matrices.
image_width: int specifying desired output image width in pixels.
image_height: int specifying desired output image height in pixels.
background_value: a 1-D float32 tensor with shape [attribute_count]. Pixels
that lie outside all triangles take this value.
Returns:
A 4-D float32 tensor with shape [batch_size, image_height, image_width,
attribute_count], containing the interpolated vertex attributes at
each pixel.
Raises:
ValueError: An invalid argument to the method is detected.
"""
if not image_width > 0:
raise ValueError('Image width must be > 0.')
if not image_height > 0:
raise ValueError('Image height must be > 0.')
if len(vertices.shape) != 3:
raise ValueError('The vertex buffer must be 3D.')
batch_size = vertices.shape[0].value
vertex_count = vertices.shape[1].value
# We map the coordinates to normalized device coordinates before passing
# the scene to the rendering kernel to keep as many ops in tensorflow as
# possible.
homogeneous_coord = tf.ones([batch_size, vertex_count, 1],
dtype=tf.float32)
vertices_homogeneous = tf.concat([vertices, homogeneous_coord], 2)
# Vertices are given in row-major order, but the transformation pipeline is
# column major:
clip_space_points = tf.matmul(vertices_homogeneous,
projection_matrices,
transpose_b=True)
# Perspective divide, first thresholding the homogeneous coordinate to avoid
# the possibility of NaNs:
clip_space_points_w = tf.maximum(
tf.abs(clip_space_points[:, :, 3:4]),
_MINIMUM_PERSPECTIVE_DIVIDE_THRESHOLD) * tf.sign(
clip_space_points[:, :, 3:4])
normalized_device_coordinates = (clip_space_points[:, :, 0:3] /
clip_space_points_w)
per_image_uncorrected_barycentric_coordinates = []
per_image_vertex_ids = []
for im in range(vertices.shape[0]):
barycentric_coords, triangle_ids, _ = (
rasterize_triangles_module.rasterize_triangles(
normalized_device_coordinates[im, :, :], triangles,
image_width, image_height))
per_image_uncorrected_barycentric_coordinates.append(
tf.reshape(barycentric_coords, [-1, 3]))
# Gathers the vertex indices now because the indices don't contain a batch
# identifier, and reindexes the vertex ids to point to a (batch,vertex_id)
vertex_ids = tf.gather(triangles, tf.reshape(triangle_ids, [-1]))
reindexed_ids = tf.add(vertex_ids, im * vertices.shape[1].value)
per_image_vertex_ids.append(reindexed_ids)
uncorrected_barycentric_coordinates = tf.concat(
per_image_uncorrected_barycentric_coordinates, axis=0)
vertex_ids = tf.concat(per_image_vertex_ids, axis=0)
# Indexes with each pixel's clip-space triangle's extrema (the pixel's
# 'corner points') ids to get the relevant properties for deferred shading.
flattened_vertex_attributes = tf.reshape(attributes,
[batch_size * vertex_count, -1])
corner_attributes = tf.gather(flattened_vertex_attributes, vertex_ids)
# Barycentric interpolation is linear in the reciprocal of the homogeneous
# W coordinate, so we use these weights to correct for the effects of
# perspective distortion after rasterization.
perspective_distortion_weights = tf.reciprocal(
tf.reshape(clip_space_points_w, [-1]))
corner_distortion_weights = tf.gather(perspective_distortion_weights,
vertex_ids)
# Apply perspective correction to the barycentric coordinates. This step is
# required since the rasterizer receives normalized-device coordinates (i.e.,
# after perspective division), so it can't apply perspective correction to the
# interpolated values.
weighted_barycentric_coordinates = tf.multiply(
uncorrected_barycentric_coordinates, corner_distortion_weights)
barycentric_reweighting_factor = tf.reduce_sum(
weighted_barycentric_coordinates, axis=1)
corrected_barycentric_coordinates = tf.divide(
weighted_barycentric_coordinates,
tf.expand_dims(tf.maximum(barycentric_reweighting_factor,
_MINIMUM_REWEIGHTING_THRESHOLD),
axis=1))
# Computes the pixel attributes by interpolating the known attributes at the
# corner points of the triangle interpolated with the barycentric coordinates.
weighted_vertex_attributes = tf.multiply(
corner_attributes,
tf.expand_dims(corrected_barycentric_coordinates, axis=2))
summed_attributes = tf.reduce_sum(weighted_vertex_attributes, axis=1)
attribute_images = tf.reshape(summed_attributes,
[batch_size, image_height, image_width, -1])
# Barycentric coordinates should approximately sum to one where there is
# rendered geometry, but be exactly zero where there is not.
alphas = tf.clip_by_value(
tf.reduce_sum(2.0 * corrected_barycentric_coordinates, axis=1), 0.0,
1.0)
alphas = tf.reshape(alphas, [batch_size, image_height, image_width, 1])
attributes_with_background = (alphas * attribute_images +
(1.0 - alphas) * background_value)
return attributes_with_background, alphas
import tensorflow as tf """tf.reciprocal(x,name=None) 功能：求x的倒数。 输入：x为张量，可以为`half`,`float32`, `float64`, `int32`, `int64`,`complex64`,`complex128`类型。""" x = tf.constant([[2, 0, -3]], tf.float64) z = tf.reciprocal(x) sess = tf.Session() print(sess.run(z)) sess.close() # z==>[[0.5 inf -0.33333333]]
def _resample_inv_dst_weighting(self, inputs, sample_coords):
# inverse distance weighting using 2^(sptial_rank) neighbours
in_size = inputs.shape
partial_shape = not in_size.is_fully_defined()
try:
batch_size = int(in_size[0])
n_coords = int(sample_coords.shape[0])
in_spatial_rank = infer_spatial_rank(inputs)
in_spatial_size = \
None if partial_shape else in_size.as_list()[1:-1]
except (TypeError, AssertionError, ValueError):
tf.logging.fatal('Unknown input shape, at least batch size '
'and rank of the inputs are required.')
raise
out_rank = len(sample_coords.get_shape())
binary_neighbour_ids = _binary_neighbour_ids(in_spatial_rank)
weight_id = [[[c, i] for i, c in enumerate(bc)]
for bc in binary_neighbour_ids]
sample_coords_shape = [out_rank - 1, 0] + list(range(1, out_rank - 1))
sample_coords = tf.transpose(sample_coords, sample_coords_shape)
if partial_shape or in_spatial_size is None:
all_coords_f = tf.stack(
[tf.floor(sample_coords), tf.ceil(sample_coords)])
else:
# broadcasting input spatial size for boundary functions
expanded_spatial_size = \
[len(in_spatial_size)] + [1] * (out_rank - 1)
b_size = tf.reshape(in_spatial_size, expanded_spatial_size)
# find floor and ceil coordinates
all_coords_f = tf.stack([
self.boundary_func(tf.floor(sample_coords), b_size),
self.boundary_func(tf.ceil(sample_coords), b_size)])
# find N weights associated to each output point
diff = tf.stack(
[tf.squared_difference(sample_coords - EPS, all_coords_f[0]),
tf.squared_difference(sample_coords + EPS, all_coords_f[1])])
# gather_nd for both matrices, the same as:
# point_weights = tf.gather_nd(diff, weight_id)
# knots_id = tf.gather_nd(all_coords_f, weight_id)
n_val = tf.gather_nd(
tf.stack([diff, all_coords_f], axis=-1), weight_id)
n_val = tf.unstack(n_val, axis=-1)
point_weights, knots_id = n_val[0], n_val[1]
# inverse distance weighting
# sum_i (w_i*p_i/(sum_j w_j)) where w_i = 1/((p-p_i)^2)
# point_weights has the shape:100
# `[N, input_rank, b, sp_dim_0, ..., sp_dim_K]`
# where:
# `N` is 2**source data spatial rank
# `b` is batch size,
# `sp_dim_0` is the output spatial output 0,
#
# `point_weights` represents (p - p_i)^2
# with i= 0...2**source_rank neighbours
# (to do: these operations could be refactored as a resampling kernel)
point_weights = tf.reduce_sum(point_weights, axis=1)
# skip this as power = 2.0:
# self.power = 2.0
# point_weights = tf.pow(point_weights, self.power / 2.0)
point_weights = tf.reciprocal(point_weights)
point_weights = point_weights / tf.reduce_sum(point_weights, axis=0)
# find N neighbours associated to each output point
# knots_shape = tf.concat([[0], tf.range(2, out_rank + 1), [1]], 0)
knots_shape = [0] + list(range(2, out_rank + 1)) + [1]
knots_id = tf.cast(knots_id, COORDINATES_TYPE)
knots_id = tf.transpose(knots_id, knots_shape)
# get values of N neighbours
batch_inputs = tf.unstack(inputs, axis=0)
batch_knots = tf.unstack(knots_id, axis=1)
if batch_size == n_coords:
samples = [tf.gather_nd(img, knot)
for (img, knot) in zip(batch_inputs, batch_knots)]
elif n_coords == 1 and batch_size > 1:
samples = [tf.gather_nd(img, batch_knots[0])
for img in batch_inputs]
else:
raise NotImplementedError
samples = tf.stack(samples, axis=1)
# weighted average over N neighbours
return tf.reduce_sum(
samples * tf.expand_dims(point_weights, axis=-1), axis=0)
def mvn_diag_from_natural_params(p1, p2):
sigmasq = 0.5 * tf.reciprocal(-p1)
mu = sigmasq * p2
return {'mu': mu, 'diag_stdev': tf.sqrt(sigmasq)}
def generalised_dice_loss(prediction,
ground_truth,
weight_map=None,
type_weight='Square'):
"""
Function to calculate the Generalised Dice Loss defined in
Sudre, C. et. al. (2017) Generalised Dice overlap as a deep learning
loss function for highly unbalanced segmentations. DLMIA 2017
:param prediction: the logits
:param ground_truth: the segmentation ground truth
:param weight_map:
:param type_weight: type of weighting allowed between labels (choice
between Square (square of inverse of volume),
Simple (inverse of volume) and Uniform (no weighting))
:return: the loss
"""
prediction = tf.cast(prediction, tf.float32)
if len(ground_truth.shape) == len(prediction.shape):
ground_truth = ground_truth[..., -1]
one_hot = labels_to_one_hot(ground_truth, tf.shape(prediction)[-1])
if weight_map is not None:
n_classes = prediction.shape[1].value
weight_map_nclasses = tf.reshape(tf.tile(weight_map, [n_classes]),
prediction.get_shape())
ref_vol = tf.sparse_reduce_sum(weight_map_nclasses * one_hot,
reduction_axes=[0])
intersect = tf.sparse_reduce_sum(weight_map_nclasses * one_hot *
prediction,
reduction_axes=[0])
seg_vol = tf.reduce_sum(tf.multiply(weight_map_nclasses, prediction),
0)
else:
ref_vol = tf.sparse_reduce_sum(one_hot, [0, 1, 2])
intersect = tf.sparse_reduce_sum(one_hot * prediction, [0, 1, 2])
seg_vol = tf.reduce_sum(prediction, [0, 1, 2])
if type_weight == 'Square':
weights = tf.reciprocal(tf.square(ref_vol))
elif type_weight == 'Simple':
weights = tf.reciprocal(ref_vol)
elif type_weight == 'Uniform':
weights = tf.ones_like(ref_vol)
elif type_weight == 'Fixed':
weights = tf.constant([
0.0006, 0.0006, 0.1656, 0.1058, 0.0532, 0.0709, 0.1131, 0.3155,
0.1748
]) #W3 = 1/sqrt(freq)
else:
raise ValueError("The variable type_weight \"{}\""
"is not defined.".format(type_weight))
new_weights = tf.where(tf.is_inf(weights), tf.zeros_like(weights), weights)
weights = tf.where(tf.is_inf(weights),
tf.ones_like(weights) * tf.reduce_max(new_weights),
weights)
generalised_dice_numerator = 2 * tf.reduce_sum(
tf.multiply(weights, intersect))
generalised_dice_denominator = tf.reduce_sum(
tf.multiply(weights, tf.maximum(seg_vol + ref_vol, 1)))
# generalised_dice_denominator = tf.reduce_sum(tf.multiply(weights, seg_vol + ref_vol)) + 1e-6
generalised_dice_score = generalised_dice_numerator / generalised_dice_denominator
generalised_dice_score = tf.where(tf.is_nan(generalised_dice_score), 1.0,
generalised_dice_score)
return 1 - generalised_dice_score
num_dim = 2
r_threshold = 0.1
time_step = 10**(-4)
r_x = r[:, 0] # x component of positions vec
r_y = r[:, 1] # y component of positions vec
outer_rx = tf.cast(tf.transpose([r_x]) - r_x, dtype=tf.float32)
outer_ry = tf.cast(tf.transpose([r_y]) - r_y, dtype=tf.float32)
square_outer_x = tf.cast(tf.square(outer_rx), dtype=tf.float32)
square_outer_y = tf.cast(tf.square(outer_ry), dtype=tf.float32)
dist_mat = tf.sqrt(square_outer_x + square_outer_y) # separation matrix
dist_mat = tf.matrix_set_diag(dist_mat, tf.ones(num_particle))
check_threshold = tf.cond(
tf.size(tf.where(tf.less_equal(dist_mat, r_threshold))) > 0,
lambda: tf.assign(flag, 1), lambda: tf.assign(flag, 0))
dist_mat = tf.reciprocal(dist_mat)
dist_mat = tf.cast(G * tf.pow(dist_mat, 3), dtype=tf.float32)
dist_mat = tf.matrix_set_diag(dist_mat, tf.zeros(num_particle))
dist_mat_x = tf.multiply(dist_mat, outer_rx)
dist_mat_y = tf.multiply(dist_mat, outer_ry)
acc_x = tf.matmul(dist_mat_x, tf.reshape(masses, [tf.size(masses), 1]))
acc_y = tf.matmul(dist_mat_y, tf.reshape(masses, [tf.size(masses), 1]))
acc_x = tf.reshape(acc_x, [tf.size(acc_x), 1])
acc_y = tf.reshape(acc_y, [tf.size(acc_y), 1])
acc_mat = tf.concat([acc_x, acc_y], 1) # acceleration matrix
update_r = tf.assign(r, r + time_step * v + (1 / 2) *
(time_step**2) * acc_mat) # update the configuration r
update_v = tf.assign(v, v + time_step * acc_mat) # update the configuration v
import time
start_time = time.time()
def _mini_batch_training_op(self, inputs, cluster_idx_list,
cluster_centers, cluster_centers_var,
total_counts):
"""Creates an op for training for mini batch case.
Args:
inputs: list of input Tensors.
cluster_idx_list: A vector (or list of vectors). Each element in the
vector corresponds to an input row in 'inp' and specifies the cluster id
corresponding to the input.
cluster_centers: Tensor of cluster centers, possibly normalized.
cluster_centers_var: Tensor Ref of cluster centers.
total_counts: Tensor Ref of cluster counts.
Returns:
An op for doing an update of mini-batch k-means.
"""
update_ops = []
for inp, cluster_idx in zip(inputs, cluster_idx_list):
with ops.colocate_with(inp):
assert total_counts is not None
cluster_idx = tf.reshape(cluster_idx, [-1])
# Dedupe the unique ids of cluster_centers being updated so that updates
# can be locally aggregated.
unique_ids, unique_idx = tf.unique(cluster_idx)
num_unique_cluster_idx = tf.size(unique_ids)
# Fetch the old values of counts and cluster_centers.
with ops.colocate_with(total_counts):
old_counts = tf.gather(total_counts, unique_ids)
with ops.colocate_with(cluster_centers):
old_cluster_centers = tf.gather(cluster_centers,
unique_ids)
# Locally aggregate the increment to counts.
count_updates = tf.unsorted_segment_sum(
tf.ones_like(unique_idx, dtype=total_counts.dtype),
unique_idx, num_unique_cluster_idx)
# Locally compute the sum of inputs mapped to each id.
# For a cluster with old cluster value x, old count n, and with data
# d_1,...d_k newly assigned to it, we recompute the new value as
# x += (sum_i(d_i) - k * x) / (n + k).
# Compute sum_i(d_i), see comment above.
cluster_center_updates = tf.unsorted_segment_sum(
inp, unique_idx, num_unique_cluster_idx)
# Shape to enable broadcasting count_updates and learning_rate to inp.
# It extends the shape with 1's to match the rank of inp.
broadcast_shape = tf.concat(0, [
tf.reshape(num_unique_cluster_idx, [1]),
tf.ones(tf.reshape(tf.rank(inp) - 1, [1]), dtype=tf.int32)
])
# Subtract k * x, see comment above.
cluster_center_updates -= tf.cast(
tf.reshape(count_updates, broadcast_shape),
inp.dtype) * old_cluster_centers
learning_rate = tf.reciprocal(
tf.cast(old_counts + count_updates, inp.dtype))
learning_rate = tf.reshape(learning_rate, broadcast_shape)
# scale by 1 / (n + k), see comment above.
cluster_center_updates *= learning_rate
# Apply the updates.
update_counts = tf.scatter_add(total_counts, unique_ids,
count_updates)
update_cluster_centers = tf.scatter_add(cluster_centers_var,
unique_ids,
cluster_center_updates)
update_ops.extend([update_counts, update_cluster_centers])
return tf.group(*update_ops)
def network(self, inputs, action, phase):
# shared net
shared_net = tf.contrib.layers.fully_connected(inputs, self.shared_layer_dim, activation_fn=None,
weights_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True),
weights_regularizer=tf.contrib.layers.l2_regularizer(0.01),
biases_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True))
shared_net = self.apply_norm(shared_net, activation_fn=tf.nn.relu, phase=phase, layer_num=1)
# action branch
action_net = tf.contrib.layers.fully_connected(shared_net, self.actor_layer_dim, activation_fn=None,
weights_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True),
# tf.truncated_normal_initializer(),
weights_regularizer=None,
# tf.contrib.layers.l2_regularizer(0.001),
biases_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True))
action_net = self.apply_norm(action_net, activation_fn=tf.nn.relu, phase=phase, layer_num=2)
action_prediction_mean = tf.contrib.layers.fully_connected(action_net, self.num_modal * self.action_dim,
activation_fn=tf.tanh,
weights_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True),
# weights_initializer=tf.random_uniform_initializer(-3e-3, 3e-3),
weights_regularizer=None,
# tf.contrib.layers.l2_regularizer(0.001),
biases_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True))
# biases_initializer=tf.random_uniform_initializer(-3e-3, 3e-3))
action_prediction_sigma = tf.contrib.layers.fully_connected(action_net, self.num_modal * self.action_dim,
activation_fn=tf.tanh,
weights_initializer=tf.random_uniform_initializer(0,1),#3e-3),
weights_regularizer=None,
# tf.contrib.layers.l2_regularizer(0.001),
biases_initializer=tf.random_uniform_initializer(
-3e-3, 3e-3))
action_prediction_alpha = tf.contrib.layers.fully_connected(action_net, self.num_modal, activation_fn=tf.tanh,
weights_initializer=tf.random_uniform_initializer(
-3e-3, 3e-3),
weights_regularizer=None,
# tf.contrib.layers.l2_regularizer(0.001),
biases_initializer=tf.random_uniform_initializer(
-3e-3, 3e-3))
# reshape output
action_prediction_mean = tf.reshape(action_prediction_mean, [-1, self.num_modal, self.action_dim])
action_prediction_sigma = tf.reshape(action_prediction_sigma, [-1, self.num_modal, self.action_dim])
action_prediction_alpha = tf.reshape(action_prediction_alpha, [-1, self.num_modal, 1])
# scale mean to env. action domain
action_prediction_mean = tf.multiply(action_prediction_mean, self.action_max)
# exp. sigma
# action_prediction_sigma = tf.exp(tf.scalar_mul(self.sigma_scale, action_prediction_sigma))
log_std = self.LOG_STD_MIN + 0.5 * (self.LOG_STD_MAX - self.LOG_STD_MIN) * (action_prediction_sigma + 1)
action_prediction_sigma = tf.exp(log_std)
# mean: [None, num_modal, action_dim] : [None, 1]
# sigma: [None, num_modal, action_dim] : [None, 1]
# alpha: [None, num_modal, 1] : [None, 1]
# compute softmax prob. of alpha
max_alpha = tf.reduce_max(action_prediction_alpha, axis=1, keepdims=True)
action_prediction_alpha = tf.subtract(action_prediction_alpha, max_alpha)
action_prediction_alpha = tf.exp(action_prediction_alpha)
normalize_alpha = tf.reciprocal(tf.reduce_sum(action_prediction_alpha, axis=1, keepdims=True))
action_prediction_alpha = tf.multiply(normalize_alpha, action_prediction_alpha)
# Q branch
q_net = tf.contrib.layers.fully_connected(tf.concat([shared_net, action], 1), self.expert_layer_dim,
activation_fn=None,
weights_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True),
# tf.truncated_normal_initializer(), \
weights_regularizer=tf.contrib.layers.l2_regularizer(0.01),
biases_initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode="FAN_IN", uniform=True))
q_net = self.apply_norm(q_net, activation_fn=tf.nn.relu, phase=phase, layer_num=3)
q_prediction = tf.contrib.layers.fully_connected(q_net, 1, activation_fn=None,
weights_initializer=tf.random_uniform_initializer(-3e-3, 3e-3),
weights_regularizer=tf.contrib.layers.l2_regularizer(0.01),
biases_initializer=tf.random_uniform_initializer(-3e-3, 3e-3))
return action_prediction_mean, action_prediction_sigma, action_prediction_alpha, q_prediction
def l1_norm(x, axis=None, epsilon=1e-6, name=None):
with tf.name_scope(name, "l1_normalize", [x]) as name:
x = tf.convert_to_tensor(x, name="x")
square_sum = tf.reduce_sum(x, axis, keepdims=True)
x_inv_norm = tf.reciprocal(tf.maximum(square_sum, epsilon))
return tf.multiply(x, x_inv_norm, name=name)
def predict(self):
embedding_users = tf.get_variable(
"embedding_users", [self.num_users, self.embedding_size])
embedding_items = tf.get_variable(
"embedding_items", [self.num_items, self.embedding_size])
self.embedding_users = embedding_users
self.embedding_items = embedding_items
#user_embedding: batch_len*embedding_size
user_embedding = tf.reduce_sum(tf.nn.embedding_lookup(
embedding_users, self.input_user),
axis=1)
item_embedding = tf.reduce_sum(tf.nn.embedding_lookup(
embedding_items, self.input_item),
axis=1)
#user_ratings: batch_len*num_items
user_ratings = tf.reduce_sum(tf.gather(self.rating_matrix,
self.input_user),
axis=1)
item_ratings = tf.reduce_sum(tf.gather(
tf.transpose(self.rating_matrix), self.input_item),
axis=1)
#user_ratings_nonzero: batch_len
user_ratings_nonzero = tf.cast(tf.reduce_sum(user_ratings, axis=1),
tf.float32)
item_ratings_nonzero = tf.cast(tf.reduce_sum(item_ratings, axis=1),
tf.float32)
#user_nsvd_embedding: batch_len*embedding_size
# user_nsvd_embedding = tf.expand_dims(tf.reciprocal(user_ratings_nonzero),
# axis=1) * tf.matmul(user_ratings, embedding_items)
# item_nsvd_embedding = tf.expand_dims(tf.reciprocal(item_ratings_nonzero),
# axis=1) * tf.matmul(item_ratings, embedding_users)
user_nsvd_embedding = tf.expand_dims(
tf.reciprocal(tf.sqrt(user_ratings_nonzero + 1)),
axis=1) * tf.matmul(user_ratings, embedding_items)
item_nsvd_embedding = tf.expand_dims(
tf.reciprocal(tf.sqrt(item_ratings_nonzero + 1)),
axis=1) * tf.matmul(item_ratings, embedding_users)
def hbijk(embed_a, embed_b, num_slice, variable_scope):
assert num_slice >= 1
with tf.variable_scope(variable_scope):
B = tf.get_variable("B_0",
[self.embedding_size, self.embedding_size])
hb = tf.expand_dims(tf.reduce_sum(tf.matmul(embed_a, B) *
embed_b,
axis=1),
axis=1)
for i in xrange(num_slice - 1):
B = tf.get_variable(
"B_%d" % (i + 1),
[self.embedding_size, self.embedding_size])
hb_this = tf.expand_dims(tf.reduce_sum(
tf.matmul(embed_a, B) * embed_b, axis=1),
axis=1)
hb = tf.concat([hb, hb_this], axis=1)
return hb
num_slice = 4
hb_u0i0 = hbijk(user_embedding, item_embedding, num_slice, "u0i0")
hb_u0i1 = hbijk(user_embedding, item_nsvd_embedding, num_slice, "u0i1")
hb_u1i0 = hbijk(user_nsvd_embedding, item_embedding, num_slice, "u1i0")
hb_u1i1 = hbijk(user_nsvd_embedding, item_nsvd_embedding, num_slice,
"u1i1")
# merge_embedding = tf.concat([user_embedding, item_embedding,
# tf.stop_gradient(user_nsvd_embedding),
# tf.stop_gradient(item_nsvd_embedding),],
# axis=1, name="merge_embedding")
merge_embedding = tf.concat([
user_embedding,
item_embedding,
user_nsvd_embedding,
item_nsvd_embedding,
],
axis=1,
name="merge_embedding")
ha = tf.contrib.slim.fully_connected(merge_embedding, 32, tf.identity)
f = tf.tanh(tf.concat([ha, hb_u0i0, hb_u0i1, hb_u1i0, hb_u1i1],
axis=1),
name="g_ha_hb")
print "g(ha,hb) shape is: (should be none*48)"
print f.get_shape().as_list()
f = tf.contrib.slim.fully_connected(f, 1, tf.identity)
# print "merge_embedding shape is:"
# print merge_embedding.get_shape().as_list()
# x=merge_embedding
# for i in xrange(len(self.layers)-1):
# x = tf.contrib.slim.fully_connected(x,self.layers[i+1] )
# x = tf.contrib.slim.fully_connected(x, 1, tf.identity)
return f
def prepare_gt(data,
from_inverse_depth = False,
to_inverse_depth = False,
disparity_map = False,
data_format='channels_last', # TODO: Support channels first is not implemented yet
focal_norm = False,
computate_sig=True,
img_keys = ['image'],
depth_keys = ['depth'], normal_keys = [],
compute_normals = True, # TODO: Support online normal estimation
focal_factor = 100.,
downsampling_depth=5,
sig_params = None,**kargs):
gt={}
if sig_params is None:
sig_params = {'deltas':[1,2,4,8,16], 'weights':[1,1,1,1,1], 'epsilon': 0.001}
K = data['intrinsics']
w = data['depth'].shape[-2].value
h = data['depth'].shape[-3].value
focal_w = K[0,0]
focal_h = K[0,1]
ppw = K[0,2]
pph = K[0,3]
print([h,w])
print([focal_w,focal_h, ppw,pph])
print([focal_w/w,focal_h/h, ppw/w,pph/h])
w,h=float(w),float(h)
intrinsics = tf.math.divide(K,tf.convert_to_tensor([[w,h,w,h]]))
# Prepare depth images at different scales:
norm_mul = 1.
if focal_norm:
focal_orig = ((K[0,0]+K[0,1])/2.)
norm_mul = focal_factor/focal_orig
if to_inverse_depth:
norm_mul = tf.reciprocal(norm_mul)
for dk in depth_keys:
depth = data[dk]*norm_mul
if (from_inverse_depth ^ to_inverse_depth): # XOR (true if different)
depth=tf.reciprocal(depth)
for i in range(downsampling_depth):
gt[dk+str(i)]=depth
if computate_sig:
gt['sig_'+dk+str(i)]=scale_invariant_gradient(depth,**sig_params)
if compute_normals:
infn = lambda x: convert_NHWC_to_NCHW(tf.expand_dims(x,0))
outfn = lambda x: tf.squeeze(convert_NCHW_to_NHWC(x),0)
gt['norm_'+dk+str(i)]=outfn(
sops.depth_to_normals(infn(depth),
intrinsics,inverse_depth=to_inverse_depth))
depth = nn_downsampling(depth,2)
# Prepare normal images at different scales:
for nk in normal_keys:
normals = data[nk]
for i in range(downsampling_depth):
gt[nk+str(i)]=normals
normals = nn_downsampling(normals,2)
data['gt']=gt
return data
def QuantizedWeight(name, x, n, nbit=2):
"""
Quantize weight.
Args:
x (tf.Tensor): a 4D tensor.
Must have known number of channels, but can have other unknown dimensions.
name (str): operator's name.
n (int or double): variance of weight initialization.
nbit (int): number of bits of quantized weight. Defaults to 2.
Returns:
tf.Tensor with attribute `variables`.
Variable Names:
* ``basis``: basis of quantized weight.
Note:
About multi-GPU training: moving averages across GPUs are not aggregated.
Batch statistics are computed by main training tower. This is consistent with most frameworks.
"""
num_filters = x.get_shape().as_list()[-1]
init_basis = []
base = NORM_PPF_0_75 * ((2. / n)**0.5) / (2**(nbit - 1))
for j in range(nbit):
init_basis.append([(2**j) * base for i in range(num_filters)])
init_basis = tf.constant_initializer(init_basis)
bit_dims = [nbit, num_filters]
num_levels = 2**nbit
delta = EPS
# initialize level multiplier
init_level_multiplier = []
for i in range(num_levels):
level_multiplier_i = [0. for j in range(nbit)]
level_number = i
for j in range(nbit):
binary_code = level_number % 2
if binary_code == 0:
binary_code = -1
level_multiplier_i[j] = float(binary_code)
level_number = level_number // 2
init_level_multiplier.append(level_multiplier_i)
# initialize threshold multiplier
init_thrs_multiplier = []
for i in range(1, num_levels):
thrs_multiplier_i = [0. for j in range(num_levels)]
thrs_multiplier_i[i - 1] = 0.5
thrs_multiplier_i[i] = 0.5
init_thrs_multiplier.append(thrs_multiplier_i)
with tf.variable_scope(name):
basis = tf.get_variable('basis',
bit_dims,
tf.float32,
initializer=init_basis,
trainable=False)
level_codes = tf.constant(init_level_multiplier)
thrs_multiplier = tf.constant(
init_thrs_multiplier
) # ValueError: Cannot create a tensor proto whose content is larger than 2GB.
sum_multiplier = tf.constant(
1., shape=[1, tf.reshape(x, [-1, num_filters]).get_shape()[0]])
sum_multiplier_basis = tf.constant(1., shape=[1, nbit])
# calculate levels and sort
levels = tf.matmul(level_codes, basis)
levels, sort_id = tf.nn.top_k(tf.transpose(levels, [1, 0]), num_levels)
levels = tf.reverse(levels, [-1])
sort_id = tf.reverse(sort_id, [-1])
levels = tf.transpose(levels, [1, 0])
sort_id = tf.transpose(sort_id, [1, 0])
# calculate threshold
thrs = tf.matmul(thrs_multiplier, levels)
# calculate level codes per channel
reshape_x = tf.reshape(x, [-1, num_filters])
level_codes_channelwise_dims = tf.stack(
[num_levels * num_filters, nbit])
level_codes_channelwise = tf.fill(level_codes_channelwise_dims, 0.)
for i in range(num_levels):
eq = tf.equal(sort_id, i)
level_codes_channelwise = tf.where(
tf.reshape(eq, [-1]), level_codes_channelwise + level_codes[i],
level_codes_channelwise)
level_codes_channelwise = tf.reshape(level_codes_channelwise,
[num_levels, num_filters, nbit])
# calculate output y and its binary code
y = tf.zeros_like(x) + levels[0] # output
zero_dims = tf.stack([tf.shape(reshape_x)[0] * num_filters, nbit])
bits_y = tf.fill(zero_dims, -1.)
zero_y = tf.zeros_like(x)
zero_bits_y = tf.fill(zero_dims, 0.)
zero_bits_y = tf.reshape(zero_bits_y, [-1, num_filters, nbit])
for i in range(num_levels - 1):
g = tf.greater(x, thrs[i])
y = tf.where(g, zero_y + levels[i + 1], y)
bits_y = tf.where(
tf.reshape(g, [-1]),
tf.reshape(zero_bits_y + level_codes_channelwise[i + 1],
[-1, nbit]), bits_y)
bits_y = tf.reshape(bits_y, [-1, num_filters, nbit])
ctx = get_current_tower_context() # current tower context
# training
if ctx.is_main_training_tower:
BT = tf.transpose(bits_y, [2, 0, 1])
# calculate BTxB
BTxB = []
for i in range(nbit):
for j in range(nbit):
BTxBij = tf.multiply(BT[i], BT[j])
BTxBij = tf.matmul(sum_multiplier, BTxBij)
if i == j:
mat_one = tf.ones([1, num_filters])
BTxBij = BTxBij + (delta * mat_one) # + E
BTxB.append(BTxBij)
BTxB = tf.reshape(tf.stack(values=BTxB), [nbit, nbit, num_filters])
# calculate inverse of BTxB
if nbit > 2:
BTxB_transpose = tf.transpose(BTxB, [2, 0, 1])
# 1) naive
# BTxB_inv = tf.matrix_inverse(BTxB_transpose)
# 2) try, except
try:
BTxB_inv = tf.matrix_inverse(BTxB_transpose,
adjoint=None,
name=None)
except:
BTxB_ttt = tf.add(
BTxB_transpose,
tf.math.scalar_mul(tf.identity((BTxB_transpose.shape)),
1e-6))
BTxB_inv = tf.matrix_inverse(BTxB_ttt,
adjoint=None,
name=None)
BTxB_inv = tf.transpose(BTxB_inv, [1, 2, 0])
elif nbit == 2:
det = tf.multiply(BTxB[0][0], BTxB[1][1]) - tf.multiply(
BTxB[0][1], BTxB[1][0])
inv = []
inv.append(BTxB[1][1] / det)
inv.append(-BTxB[0][1] / det)
inv.append(-BTxB[1][0] / det)
inv.append(BTxB[0][0] / det)
BTxB_inv = tf.reshape(tf.stack(values=inv),
[nbit, nbit, num_filters])
elif nbit == 1:
BTxB_inv = tf.reciprocal(BTxB)
# calculate BTxX
BTxX = []
for i in range(nbit):
BTxXi0 = tf.multiply(BT[i], reshape_x)
BTxXi0 = tf.matmul(sum_multiplier, BTxXi0)
BTxX.append(BTxXi0)
BTxX = tf.reshape(tf.stack(values=BTxX), [nbit, num_filters])
BTxX = BTxX + (delta * basis) # + basis
# calculate new basis
new_basis = []
for i in range(nbit):
new_basis_i = tf.multiply(BTxB_inv[i], BTxX)
new_basis_i = tf.matmul(sum_multiplier_basis, new_basis_i)
add_moving_summary(
tf.reduce_mean(new_basis_i, name='new_basis_bit' + str(i)))
new_basis.append(new_basis_i)
new_basis = tf.reshape(tf.stack(values=new_basis),
[nbit, num_filters])
# create moving averages op
updata_moving_basis = moving_averages.assign_moving_average(
basis, new_basis, MOVING_AVERAGES_FACTOR)
add_model_variable(basis)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, updata_moving_basis)
# add_moving_summary(tf.identity(basis, name='basis'), tf.identity(new_basis, name='basis_new'))
# add_moving_summary(tf.identity(basis, name='basis'))
y = x + tf.stop_gradient(-x) + tf.stop_gradient(y) # gradient: y=x
y.variables = VariableHolder(basis=basis)
return y
def normal_from_natural_params(p1, p2):
sigmasq = 0.5 * tf.reciprocal(-p1)
mu = sigmasq * p2
return {'mu': mu, 'sigma': tf.sqrt(sigmasq)}
def dsfa(xtrain, ytrain, xtest, ytest, net_shape=None, args=None):
train_num = np.shape(xtrain)[0]
bands = np.shape(xtrain)[-1]
tf.reset_default_graph()
activation = tf.nn.softsign
xd = tf.placeholder(dtype=tf.float32, shape=[None, bands])
yd = tf.placeholder(dtype=tf.float32, shape=[None, bands])
# fc1
fc1w1 = tf.Variable(
tf.truncated_normal(shape=[bands, net_shape[0]],
dtype=tf.float32,
stddev=1e-1))
fc1w2 = tf.Variable(
tf.truncated_normal(shape=[bands, net_shape[0]],
dtype=tf.float32,
stddev=1e-1))
fc1b1 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[0]], dtype=tf.float32))
fc1b2 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[0]], dtype=tf.float32))
fc1x = tf.nn.bias_add(tf.matmul(xd, fc1w1), fc1b1)
fc1y = tf.nn.bias_add(tf.matmul(yd, fc1w2), fc1b2)
fc11 = activation(fc1x)
fc12 = activation(fc1y)
# fc2
fc2w1 = tf.Variable(
tf.truncated_normal(shape=[net_shape[0], net_shape[1]],
dtype=tf.float32,
stddev=1e-1))
fc2w2 = tf.Variable(
tf.truncated_normal(shape=[net_shape[0], net_shape[1]],
dtype=tf.float32,
stddev=1e-1))
fc2b1 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[1]], dtype=tf.float32))
fc2b2 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[1]], dtype=tf.float32))
fc2x = tf.nn.bias_add(tf.matmul(fc11, fc2w1), fc2b1)
fc2y = tf.nn.bias_add(tf.matmul(fc12, fc2w2), fc2b2)
fc21 = activation(fc2x)
fc22 = activation(fc2y)
# fc3
fc3w1 = tf.Variable(
tf.truncated_normal(shape=[net_shape[1], net_shape[2]],
dtype=tf.float32,
stddev=1e-1))
fc3w2 = tf.Variable(
tf.truncated_normal(shape=[net_shape[1], net_shape[2]],
dtype=tf.float32,
stddev=1e-1))
fc3b1 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[2]], dtype=tf.float32))
fc3b2 = tf.Variable(
tf.constant(1e-1, shape=[net_shape[2]], dtype=tf.float32))
fc3x = tf.nn.bias_add(tf.matmul(fc21, fc3w1), fc3b1)
fc3y = tf.nn.bias_add(tf.matmul(fc22, fc3w2), fc3b2)
fc3x = activation(fc3x)
fc3y = activation(fc3y)
#fc3x - tf.cast(tf.divide(1, bands), tf.float32) * tf.matmul(fc3x, tf.ones([bands, bands]))
m = tf.shape(fc3x)[1]
fc_x = fc3x - tf.reduce_mean(fc3x, axis=0)
fc_y = fc3y - tf.reduce_mean(fc3y, axis=0)
Differ = fc_x - fc_y
A = tf.matmul(Differ, Differ, transpose_a=True)
A = A / train_num
sigmaX = tf.matmul(fc_x, fc_x, transpose_a=True)
sigmaY = tf.matmul(fc_y, fc_y, transpose_a=True)
sigmaX = sigmaX / train_num + args.reg * tf.eye(net_shape[-1])
sigmaY = sigmaY / train_num + args.reg * tf.eye(net_shape[-1])
B = (sigmaX + sigmaY) / 2 # + args.reg * tf.eye(net_shape[-1])
# B_inv, For numerical stability.
D_B, V_B = tf.self_adjoint_eig(B)
idx = tf.where(D_B > 1e-12)[:, 0]
D_B = tf.gather(D_B, idx)
V_B = tf.gather(V_B, idx, axis=1)
B_inv = tf.matmul(tf.matmul(V_B, tf.diag(tf.reciprocal(D_B))),
tf.transpose(V_B))
sigma = tf.matmul(B_inv, A) #+ args.reg * tf.eye(net_shape[-1])
D, V = tf.self_adjoint_eig(sigma)
#loss = tf.sqrt(tf.trace(tf.matmul(sigma,sigma)))
loss = tf.trace(tf.matmul(sigma, sigma))
optimizer = tf.train.GradientDescentOptimizer(args.lr).minimize(loss)
init = tf.global_variables_initializer()
loss_log = []
gpu_options = tf.GPUOptions(allow_growth=True)
conf = tf.ConfigProto(gpu_options=gpu_options)
sess = tf.Session(config=conf)
sess.run(init)
#writer = tf.summary.FileWriter('graph')
#writer.add_graph(sess.graph)
for k in range(args.epoch):
sess.run(optimizer, feed_dict={xd: xtrain, yd: ytrain})
if k % 100 == 0:
ll = sess.run(loss, feed_dict={xd: xtrain, yd: ytrain})
ll = ll / net_shape[-1]
logging.info('The %4d-th epochs, loss is %4.4f ' % (k, ll))
loss_log.append(ll)
matV = sess.run(V, feed_dict={xd: xtest, yd: ytest})
bVal = sess.run(B, feed_dict={xd: xtest, yd: ytest})
fcx = sess.run(fc_x, feed_dict={xd: xtest, yd: ytest})
fcy = sess.run(fc_y, feed_dict={xd: xtest, yd: ytest})
sess.close()
print('')
return loss_log, matV, fcx, fcy, bVal
[5,6], [1,4], [5,6], [3,4], [7,10] ]) real_values=np.array([[1,0,0,1,0,0,1,0]]).T x=tf.placeholder(tf.float32,[None,2]) w=tf.Variable(tf.zeros([2,2])) b=tf.Variable(tf.zeros([2])) predicted=tf.add(tf.matmul(x,w),b) y__= tf.reciprocal(1 + tf.exp(-predicted)) y=tf.placeholder(tf.float32,[None,1]) #hyperparameters learning_rate=0.01 training_epochs=20000 display_step=50 n_samples=real_values.size #cost=tf.nn.sigmoid_cross_entropy_with_logits(predicted, y) cost=tf.reduce_sum(tf.pow(y-y__,2))/(2*n_samples) optimizer=tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
def __init__(self, predicates, mutExPreds, clauses, kbLabel, save_path=""):
with tf.variable_scope('KnowledgeBase' + kbLabel) as sc:
self.clauses = clauses
if not self.clauses:
self.tensor = tf.constant(1.0)
else:
weights_tensor = tf.stack([cl.weight for cl in clauses])
clauses_value_tensor = tf.stack([cl.tensor for cl in clauses],
0)
if config.CLAUSE_AGGREGATOR == "min":
print("clauses aggregator is min")
self.tensor = tf.reduce_min(clauses_value_tensor)
if config.CLAUSE_AGGREGATOR == "mean":
self.tensor = tf.reduce_mean(clauses_value_tensor)
if config.CLAUSE_AGGREGATOR == "hmean":
self.tensor = tf.div(
tf.to_float(tf.size(clauses_value_tensor)),
tf.reduce_sum(tf.reciprocal(clauses_value_tensor),
keep_dims=True))
if config.CLAUSE_AGGREGATOR == "wmean":
self.tensor = tf.div(
tf.reduce_sum(
tf.multiply(weights_tensor, clauses_value_tensor)),
tf.reduce_sum(weights_tensor))
if config.CLAUSE_AGGREGATOR == 'log-likelihood':
# Smartly handle exp/log functions as it already uses exp sum log trick to compute product norm.
if config.FORALL_AGGREGATOR == 'product' or config.FORALL_AGGREGATOR == 'mean-log-likelihood':
self.tensor = tf.reduce_mean(clauses_value_tensor)
else:
self.tensor = tf.reduce_mean(
tf.log(clauses_value_tensor))
if config.CLAUSE_AGGREGATOR == 'w-log-likelihood':
# Smartly handle exp/log functions as it already uses exp sum log trick to compute product norm.
if config.FORALL_AGGREGATOR == 'product' or config.FORALL_AGGREGATOR == 'mean-log-likelihood':
self.tensor = tf.reduce_mean(
tf.multiply(
clauses_value_tensor, weights_tensor /
tf.reduce_sum(weights_tensor)))
else:
self.tensor = tf.reduce_mean(
tf.multiply(tf.log(clauses_value_tensor),
weights_tensor))
self.tensor = tf.reshape(self.tensor,
shape=(),
name=kbLabel + 'loss')
tf.summary.scalar(kbLabel + 'loss',
self.tensor,
collections=['train'])
self.parameters = [
param for pred in predicates for param in pred.parameters
]
self.parameters += [
param for mutPred in mutExPreds for param in mutPred.parameters
]
self.omega = tf.concat(
[tf.reshape(par, [-1]) for par in self.parameters], 0)
self.omega = tf.reshape(
self.omega, [-1]) # Completely flatten the parameter array
self.num_params = tf.shape(self.omega)
self.prior_mean = tf.placeholder("float",
shape=[
None,
],
name="prior_mean")
self.prior_lambda = tf.placeholder("float",
shape=(),
name='prior_lambda')
self.L2_regular = tf.reduce_sum(
tf.square(self.omega - self.prior_mean)) * self.prior_lambda
tf.summary.scalar(kbLabel + 'regularization',
self.L2_regular,
collections=['train'])
if config.POSITIVE_FACT_PENALTY != 0:
self.loss = self.L2_regular + \
tf.multiply(config.POSITIVE_FACT_PENALTY, self.penalize_positive_facts()) - \
PR(self.tensor)
else:
self.loss = self.L2_regular - self.tensor #PR(self.tensor)
self.save_path = save_path
self.train_op = train_op(self.loss, config.OPTIMIZER)
def attention(self):
with tf.variable_scope("curent_nl"):
inputs = tf.nn.embedding_lookup(
self.embedding_matrix, self.current_nl
) # [batch_size, self.max_seq_len, self.embedding_dim]
lstm_fw_cell = rnn.BasicLSTMCell(self.hidden_size)
lstm_bw_cell = rnn.BasicLSTMCell(self.hidden_size)
_, final_states = tf.nn.bidirectional_dynamic_rnn(
lstm_fw_cell,
lstm_bw_cell,
inputs,
sequence_length=self.current_nl_len,
dtype=tf.float32)
final_fw = tf.concat(final_states[0], axis=1)
final_bw = tf.concat(final_states[1], axis=1)
nl_outputs = tf.concat(
[final_fw, final_bw],
axis=1) # concatenate forward and backward final states
nl_outputs = tf.layers.dense(
inputs=nl_outputs,
units=self.intent_dim,
kernel_initializer=tf.random_normal_initializer,
bias_initializer=tf.random_normal_initializer)
with tf.variable_scope("history_tourist_rnn"):
lstm_fw_cell = rnn.BasicLSTMCell(self.hidden_size)
lstm_bw_cell = rnn.BasicLSTMCell(self.hidden_size)
_, final_states = tf.nn.bidirectional_dynamic_rnn(
lstm_fw_cell,
lstm_bw_cell,
self.tourist_input_intent,
dtype=tf.float32)
final_fw = tf.concat(final_states[0], axis=1)
final_bw = tf.concat(final_states[1], axis=1)
tourist_outputs = tf.concat([final_fw, final_bw], axis=1)
with tf.variable_scope("history_guide_rnn"):
lstm_fw_cell = rnn.BasicLSTMCell(self.hidden_size)
lstm_bw_cell = rnn.BasicLSTMCell(self.hidden_size)
_, final_states = tf.nn.bidirectional_dynamic_rnn(
lstm_fw_cell,
lstm_bw_cell,
self.guide_input_intent,
dtype=tf.float32)
final_fw = tf.concat(final_states[0], axis=1)
final_bw = tf.concat(final_states[1], axis=1)
guide_outputs = tf.concat([final_fw, final_bw], axis=1)
normalized_weight = tf.unstack(tf.nn.softmax(
tf.reciprocal(
tf.concat([
tf.reduce_min(self.tourist_dist, axis=1, keep_dims=True),
tf.reduce_min(self.guide_dist, axis=1, keep_dims=True)
],
axis=1))),
axis=1)
tourist_outputs = tf.multiply(
tourist_outputs, tf.expand_dims(normalized_weight[0], axis=1))
guide_outputs = tf.multiply(
guide_outputs, tf.expand_dims(normalized_weight[1], axis=1))
tourist_dense = tf.layers.dense(
inputs=tf.concat([tourist_outputs, nl_outputs], axis=1),
units=1,
kernel_initializer=tf.random_normal_initializer,
bias_initializer=tf.random_normal_initializer,
name="dense_layer")
guide_dense = tf.layers.dense(
inputs=tf.concat([guide_outputs, nl_outputs], axis=1),
units=1,
kernel_initializer=tf.random_normal_initializer,
bias_initializer=tf.random_normal_initializer,
name="dense_layer",
reuse=True)
weight = tf.unstack(tf.nn.softmax(
tf.concat([tourist_dense, guide_dense], axis=1)),
axis=1)
assert len(weight) == 2
return tf.add(
tf.multiply(tf.expand_dims(weight[0], axis=1), tourist_outputs),
tf.multiply(tf.expand_dims(weight[1], axis=1), guide_outputs))
# output layer
Wo, bo, out = gen_layer("output", out, (layer[-1], layer[-1]), activation=None)
with name_scp("mixture_model"):
pi, sigma, mu = tf.split(out, 3, axis=1, name="mixture")
new_shape = (-1, mixtures, futHor)
with name_scp("Pi"):
pi = tf.reshape(pi, shape=new_shape, name="fit_horizont")
max_pi = tf.reduce_max(pi, 1, keep_dims=True)
pi = tf.subtract(pi, max_pi)
pi = tf.exp(pi)
norm = tf.reduce_sum(pi, 1, keep_dims=True)
norm = tf.reciprocal(norm)
pi = tf.multiply(norm, pi)
with name_scp("Sigma"):
sigma = tf.reshape(sigma, shape=new_shape, name="fit_horizont")
sigma = tf.exp(sigma, name="exp")
with name_scp("Mu"):
mu = tf.reshape(mu, shape=new_shape, name="fit_horizont")
# normalisation factor for gaussian, not needed.
norm_fac = 1. / np.sqrt(2. * np.pi)
# make it feedable for tensorflow
gauss_norm = tf.constant(np.float32(norm_fac), name="Gaussnormalizer")
# don't forget to normalize over mixtures
mix_norm = tf.constant(np.float32(1 / (mixtures * futHor)), name="Mixturenorm")
def get_network_model(self, pointclouds_pl,input_label, cat_num, part_num, \
batch_size, num_point, weight_decay,is_training, bn_decay=None):
with_bn = self.conf.get_bool('with_bn')
batch_size = pointclouds_pl.get_shape()[0].value
num_point = pointclouds_pl.get_shape()[1].value
ps_function_pl = tf.concat([
pointclouds_pl,
tf.ones(shape=[batch_size, num_point, 1], dtype=tf.float32)
],
axis=2)
pool_sizes_sigma = self.conf.get_list('pool_sizes_sigma') # [256,64,4]
spacing = np.float32(self.conf.get_float('kernel_spacing'))
# first scale
network = ps_function_pl
input_channel = 4
blocks = self.conf.get_list('blocks_out_channels')
pointclouds_pl_down = []
for block_index, block in enumerate(blocks):
block_elm = ConvElements(
pointclouds_pl,
np.float32(1.) /
np.sqrt(pointclouds_pl.get_shape()[1].value, dtype=np.float32),
spacing,
np.float32(self.conf.get_float('kernel_sigma_factor')))
for out_index, out_channel in enumerate(block):
network = ConvLayer(
input_channel, block_elm, out_channel,
'{0}_block_{1}'.format(block_index, out_index),
is_training).get_layer(network, with_bn, bn_decay, False,
True)
input_channel = out_channel
pointclouds_pl_down.append([pointclouds_pl, network])
pointclouds_pl, network = PoolingLayer(
block_elm, out_channel, out_channel,
int(pool_sizes_sigma[block_index + 1][0])).get_layer(
network,
use_fps=tf.constant(False),
is_subsampling=self.conf.get_bool("is_subsampling"))
pointclouds_pl_down.append([pointclouds_pl, None])
one_hot_label_expand = tf.tile(
tf.expand_dims(input_label, axis=1),
[1, pointclouds_pl_down[-1][0].get_shape()[1].value, 1])
network = tf.concat(axis=2, values=[network, one_hot_label_expand])
for i in range(len(pointclouds_pl_down) - 1, 0, -1):
deconvnet = DeconvLayer(pointclouds_pl_down[i][0], network,
pointclouds_pl_down[i - 1][0]).get_layer()
block_elm = ConvElements(
pointclouds_pl_down[i - 1][0],
tf.reciprocal(
tf.sqrt(
1.0 *
pointclouds_pl_down[i - 1][0].get_shape()[1].value)),
spacing, self.conf.get_float('kernel_sigma_factor'))
convnet = ConvLayer(
deconvnet.get_shape()[-1].value, block_elm,
pointclouds_pl_down[i - 1][1].get_shape()[-1].value,
"{0}_deconv".format(i),
is_training).get_layer(deconvnet, with_bn, bn_decay, False,
True)
network = tf.concat([pointclouds_pl_down[i - 1][1], convnet],
axis=2)
if (self.conf.get_bool('is_dropout') and i <= 2):
network = tf_util.dropout(
network,
is_training,
"dropout_{0}".format(i),
keep_prob=self.conf.get_float('dropout.keep_prob'))
block_elm = ConvElements(pointclouds_pl_down[0][0],tf.reciprocal(tf.sqrt(1.0 * pointclouds_pl_down[0][0].get_shape()[1].value)),
spacing,
self.conf.get_float('kernel_sigma_factor')) \
network = ConvLayer(network.get_shape()[-1].value, block_elm, part_num,
"{0}_deconv".format('last'),
is_training).get_layer(network, False, bn_decay,
False, False)
return network
def normal_from_natural_params(p1, p2):
sigmasq = 0.5 * tf.reciprocal(-p1)
loc = sigmasq * p2
return {'loc': loc, 'scale': tf.sqrt(sigmasq)}
def _apply_dropout_mask(tensor_shape, keep_prob=1.0, normalize=True):
random_tensor = keep_prob + tf.random_uniform(tensor_shape, dtype=tf.float32)
binary_mask = tf.floor(random_tensor)
if normalize:
binary_mask = tf.reciprocal(keep_prob) * binary_mask
return binary_mask
def get_stroke(eos_predicted):
eos_predicted = tf.reciprocal(1+tf.exp(eos_predicted))
return eos_predicted
def normal_from_natural_params(p1, p2):
sigmasq = 0.5 * tf.reciprocal(-p1)
loc = sigmasq * p2
return {'loc': loc, 'scale': tf.sqrt(sigmasq)}
def _mini_batch_training_op(self, inputs, cluster_idx_list,
cluster_centers, cluster_centers_var,
total_counts):
"""Creates an op for training for mini batch case.
Args:
inputs: list of input Tensors.
cluster_idx_list: A vector (or list of vectors). Each element in the
vector corresponds to an input row in 'inp' and specifies the cluster id
corresponding to the input.
cluster_centers: Tensor of cluster centers, possibly normalized.
cluster_centers_var: Tensor Ref of cluster centers.
total_counts: Tensor Ref of cluster counts.
Returns:
An op for doing an update of mini-batch k-means.
"""
update_ops = []
for inp, cluster_idx in zip(inputs, cluster_idx_list):
with ops.colocate_with(inp):
assert total_counts is not None
cluster_idx = tf.reshape(cluster_idx, [-1])
# Dedupe the unique ids of cluster_centers being updated so that updates
# can be locally aggregated.
unique_ids, unique_idx = tf.unique(cluster_idx)
num_unique_cluster_idx = tf.size(unique_ids)
# Fetch the old values of counts and cluster_centers.
with ops.colocate_with(total_counts):
old_counts = tf.gather(total_counts, unique_ids)
with ops.colocate_with(cluster_centers):
old_cluster_centers = tf.gather(cluster_centers, unique_ids)
# Locally aggregate the increment to counts.
count_updates = tf.unsorted_segment_sum(
tf.ones_like(unique_idx, dtype=total_counts.dtype),
unique_idx,
num_unique_cluster_idx)
# Locally compute the sum of inputs mapped to each id.
# For a cluster with old cluster value x, old count n, and with data
# d_1,...d_k newly assigned to it, we recompute the new value as
# x += (sum_i(d_i) - k * x) / (n + k).
# Compute sum_i(d_i), see comment above.
cluster_center_updates = tf.unsorted_segment_sum(
inp,
unique_idx,
num_unique_cluster_idx)
# Shape to enable broadcasting count_updates and learning_rate to inp.
# It extends the shape with 1's to match the rank of inp.
broadcast_shape = tf.concat(
0,
[tf.reshape(num_unique_cluster_idx, [1]),
tf.ones(tf.reshape(tf.rank(inp) - 1, [1]), dtype=tf.int32)])
# Subtract k * x, see comment above.
cluster_center_updates -= tf.cast(
tf.reshape(count_updates, broadcast_shape),
inp.dtype) * old_cluster_centers
learning_rate = tf.reciprocal(tf.cast(old_counts + count_updates,
inp.dtype))
learning_rate = tf.reshape(learning_rate, broadcast_shape)
# scale by 1 / (n + k), see comment above.
cluster_center_updates *= learning_rate
# Apply the updates.
update_counts = tf.scatter_add(
total_counts,
unique_ids,
count_updates)
update_cluster_centers = tf.scatter_add(
cluster_centers_var,
unique_ids,
cluster_center_updates)
update_ops.extend([update_counts, update_cluster_centers])
return tf.group(*update_ops)
def fast_attention(q, k, v,
dim_heads,
nb_features=256,
redraw_projection=True,
ortho_scaling=0,
lm_type='bi', # unibi, bi, plm
out_proj_mat=False,
renormalize_attention=True,
numerical_stabilizer=1e-6):
'''
:param q: shape # [batch_size, num_heads, len, head_dims]
:param k: same shape with q
:param v: same shape with q
:param dim_heads: head_dims
:param nb_features: dimension of projection matrix
:param redraw_projection: use random projection matrix in each mini-batch
:param ortho_scaling:
:param lm_type: type of attention
:param out_proj_mat: is or not output projection matrix
:param renormalize_attention: (very important)
:param numerical_stabilizer:
:return:
'''
# q = tf.saturate_cast(q, tf.float32)
# k = tf.saturate_cast(k, tf.float32)
# v = tf.saturate_cast(v, tf.float32)
if redraw_projection:
# random gaussian orthogonal random matrix for every training iteration
projection_matrix = gaussian_orthogonal_random_matrix(nb_rows=nb_features,
nb_columns=dim_heads,
scaling=ortho_scaling,
dtype=q.dtype)
# print("redraw")
else:
# fixed gaussian orthogonal random matrix for every training iteration
projection_matrix = np_gaussian_orthogonal_random_matrix(nb_rows=nb_features,
nb_columns=dim_heads,
scaling=ortho_scaling,
dtype=q.dtype)
# print("no-redraw")
create_kernel = partial(nonnegative_softmax_kernel_feature_creator,
projection_matrix=projection_matrix, eps=numerical_stabilizer)
q_prime = create_kernel(q, is_query=True) # [bs, num_heads, len, head_dims]
k_prime = create_kernel(k, is_query=False)
if lm_type == 'bi':
out = linear_attention(q_prime, k_prime, v)
if not renormalize_attention:
if out_proj_mat:
return (out, projection_matrix)
else:
return out
else:
# Construct T = (K^{'})^{T} 1_L
T = tf.reduce_sum(k_prime, axis=2,
keep_dims=False) # [bs, num_heads, len, head_dims] -> [bs, num_heads, head_dims]
# Construct partition function: R = Q^{'} T = Q^{'}(K^{'})^{T} 1_L
R = tf.einsum('...nd,...d->...n', q_prime, T)
elif lm_type == 'unibi':
out = causal_linear_attention(q_prime, k_prime, v)
if not renormalize_attention:
if out_proj_mat:
return (out, projection_matrix)
else:
return out
else:
R = _denominator(q_prime, k_prime)
elif lm_type == 'plm':
NotImplementedError("Need to implement")
R_shape = shape_list(R)
R_zero_mask = tf.zeros(R_shape, dtype=R.dtype)
R_numerical_stabilizer_mask = R_zero_mask + 2*numerical_stabilizer
# R_add_numerical_stabilizer = tf.where(tf.abs(R) <= numerical_stabilizer, 2*numerical_stabilizer, 0.)
R_add_numerical_stabilizer = tf.where(tf.abs(R) <= numerical_stabilizer, R_numerical_stabilizer_mask, R_zero_mask)
R = R + R_add_numerical_stabilizer
R = tf.expand_dims(tf.reciprocal(R), axis=-1) # [bs, num_heads, len] -> [bs, num_heads, len, 1]
out = out * R
# out = tf.saturate_cast(out, tf.float16)
# [bs, num_heads, len, head_dims]
if out_proj_mat:
return (out, projection_matrix)
else:
return out
def __init__(self, config):
# Hyper parameters
num_layers = config['num_layers']
hidden_size = config['hidden_size']
max_grad_norm = config['max_grad_norm']
batch_size = config['batch_size']
sl = config['sl']
mixtures = config['mixtures']
crd = config['crd']
learning_rate = config['learning_rate']
MDN = config['MDN']
self.sl = sl
self.crd = crd
self.batch_size = batch_size
# Nodes for the input variables
self.x = tf.placeholder(dtype=tf.float32,
shape=[batch_size, crd, sl],
name='Input_data')
self.y_ = tf.placeholder(dtype=tf.int64,
shape=[batch_size],
name='Ground_truth')
self.keep_prob = tf.placeholder("float")
with tf.name_scope("LSTM") as scope:
cell = tf.nn.rnn_cell.MultiRNNCell([
lstm_cell(hidden_size, self.keep_prob)
for _ in range(num_layers)
])
inputs = tf.unstack(self.x, axis=2)
print("input shape is: ", len(inputs))
print("input shape element is: ", inputs)
outputs, _ = tf.nn.static_rnn(cell, inputs, dtype=tf.float32)
# outputs, _ = tf.contrib.rnn.static_rnn(cell, inputs, dtype=tf.float32)
# TEST
print("The length of output: ", len(outputs))
print("The elements of output: ", outputs)
with tf.name_scope("SoftMax") as scope:
final = outputs[-1]
W_c = tf.Variable(tf.random_normal([hidden_size, 2], stddev=0.01))
b_c = tf.Variable(tf.constant(0.1, shape=[2]))
self.h_c = tf.matmul(final, W_c) + b_c
print("The shape of h_c is: ", self.h_c.shape)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.h_c, labels=self.y_)
self.cost = tf.reduce_mean(loss)
loss_summary = tf.summary.scalar("cross entropy_loss", self.cost)
with tf.name_scope("Output_MDN") as scope:
params = 8 # 7+theta
# Two for distribution over hit&miss, params for distribution parameters
output_units = mixtures * params
print("The number of output units is: ", output_units)
W_o = tf.Variable(
tf.random_normal([hidden_size, output_units], stddev=0.01))
b_o = tf.Variable(tf.constant(0.5, shape=[output_units]))
# For comparison with XYZ, only up to last time_step
# --> because for final time_step you cannot make a prediction
output = outputs[:-1]
outputs_tensor = tf.concat(
output,
axis=0) # shape is [batch_size * (seq_len-1), hidden_size]
# x = tf.unstack(x, axis=1)
h_out_tensor = tf.nn.xw_plus_b(
outputs_tensor, W_o,
b_o) # size: [batch_size * (seq_len-1), output_units]
with tf.name_scope('MDN_over_next_vector') as scope:
# Next two lines are rather ugly, But its the most efficient way to reshape the data
h_xyz = tf.reshape(h_out_tensor,
(sl - 1, batch_size, output_units))
# transpose to [batch_size, output_units, sl-1]
h_xyz = tf.transpose(h_xyz, [1, 2, 0])
print("The size of h_xyz is: ", h_xyz.shape)
# x_next = tf.slice(x,[0,0,1],[batch_size,3,sl-1]) #in size [batch_size, output_units, sl-1]
x_next = tf.subtract(self.x[:, :3, 1:],
self.x[:, :3, :sl -
1]) # offset of the coordinates?
# From here, many variables have size [batch_size, mixtures, sl-1]
# xn1 size: [batch_size, x, sl-1]; xn2 size: [batch_size, y, sl-1]; xn3 size: [batch_size, z, sl-1]
xn1, xn2, xn3 = tf.split(value=x_next,
num_or_size_splits=3,
axis=1)
self.mu1, self.mu2, self.mu3, self.s1, self.s2, self.s3, self.rho, self.theta = \
tf.split(value=h_xyz, num_or_size_splits=params, axis=1)
# make the theta mixtures
# softmax all the theta's:
max_theta = tf.reduce_max(self.theta, 1, keep_dims=True)
self.theta = tf.subtract(self.theta, max_theta)
self.theta = tf.exp(self.theta)
normalize_theta = tf.reciprocal(
tf.reduce_sum(self.theta, 1, keep_dims=True))
self.theta = tf.multiply(normalize_theta, self.theta)
# Deviances are non-negative and tho between -1 and 1
self.s1 = tf.exp(self.s1)
self.s2 = tf.exp(self.s2)
self.s3 = tf.exp(self.s3)
self.rho = tf.tanh(self.rho)
# probability in x1x2 plane
px1x2 = tf_2d_normal(xn1, xn2, self.mu1, self.mu2, self.s1,
self.s2, self.rho)
px3 = tf_1d_normal(xn3, self.mu3, self.s3)
px1x2x3 = tf.multiply(px1x2, px3)
# Sum along the mixtures in dimension 1
px1x2x3_mixed = tf.reduce_sum(tf.multiply(px1x2x3, self.theta), 1)
print('You are using %.0f mixtures' % mixtures)
# at the beginning, some errors are exactly zero.
loss_seq = -tf.log(tf.maximum(px1x2x3_mixed, 1e-20))
self.cost_seq = tf.reduce_mean(loss_seq)
self.cost_comb = self.cost
if MDN:
# The magic line where both heads come together.
self.cost_comb += self.cost_seq
with tf.name_scope("train") as scope:
train_vars = tf.trainable_variables()
# We clip the gradients to prevent explosion
grads = tf.gradients(self.cost_comb, train_vars)
grads, _ = tf.clip_by_global_norm(grads, 0.5)
# Some decay on the learning rate
global_step = tf.Variable(0, trainable=False)
lr = tf.train.exponential_decay(learning_rate,
global_step,
14000,
0.95,
staircase=True)
optimizer = tf.train.AdamOptimizer(lr)
gradients = zip(grads, train_vars)
self.train_step = optimizer.apply_gradients(
gradients, global_step=global_step)
# The following block plots for every trainable variable
# - Histogram of the entries of the Tensor
# - Histogram of the gradient over the Tensor
# - Histogram of the grradient-norm over the Tensor
self.numel = tf.constant([[0]])
for gradient, variable in gradients:
if isinstance(gradient, ops.IndexedSlices):
grad_values = gradient.values
else:
grad_values = gradient
self.numel += tf.reduce_sum(tf.size(variable))
#
# h1 = tf.histogram_summary(variable.name, variable)
# h2 = tf.histogram_summary(variable.name + "/gradients", grad_values)
# h3 = tf.histogram_summary(variable.name + "/gradient_norm", clip_ops.global_norm([grad_values]))
with tf.name_scope("Evaluating_accuracy") as scope:
correct_prediction = tf.equal(tf.argmax(self.h_c, 1), self.y_)
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction,
"float"))
accuracy_summary = tf.summary.scalar("accuracy", self.accuracy)
# Define one op to call all summaries
self.merged = tf.summary.merge_all()
def tf_normal(y, mu, sigma):
result = tf.subtract(y, mu)
result = tf.multiply(result,tf.reciprocal(sigma))
result = -tf.square(result)/2
return tf.multiply(tf.exp(result),tf.reciprocal(sigma))*oneDivSqrtTwoPI
def PN_phases(freq, mTot, eta, phase_order):
"""
PN phase orders for TaylorF2
"""
#constants in equation
piM = tf.multiply(mTot * 4.92549102554e-6, np.pi) # Total mass times pi
v = tf.pow(tf.multiply(piM, freq),
(1 / 3)) # characteristic velocity of binary
etasq = tf.pow(eta, 2) # square of the symmetric mass ratio
etacb = tf.pow(eta, 3) # cube of the symmetric mass ratio
gamma = 0.577215664901532 #Euler–Mascheroni constant
#v parameters
v0 = tf.pow(v, -5)
v2 = tf.pow(v, 2)
v3 = tf.pow(v, 3)
v4 = tf.pow(v, 4)
v5 = tf.pow(v, 5)
v5log = tf.log(v)
v6 = tf.pow(v, 6)
v6log = tf.log(4 * v)
v7 = tf.pow(v, 7)
#produce PN coeffiecients
P0 = tf.multiply((3. / 128), tf.reciprocal(eta))
P2 = tf.multiply(tf.add(743 / 84, ((11) * eta)), (5 / 9))
P3 = (-16 * np.pi)
P4 = tf.multiply(
tf.add((3058673 / 1016064),
tf.add(((5429 / 1008) * eta), (617 / 144) * etasq)), (10))
P5 = tf.multiply(tf.add(7729 / 84, (-13 * eta)), (np.pi * 5 / 9))
Pl5 = tf.multiply(tf.add(7729 / 84, (-13 * eta)), (np.pi * 5 / 3))
P6 = tf.add(((11583231236531/4694215680)-(640*np.pi*np.pi/3)-(6848*gamma/21)),\
tf.add(((((-15737765635/3048192)+(2255*np.pi*np.pi/12)))*eta),\
tf.add(((76055/1728)*etasq),(-127825/1296)*etacb)))
Pl6 = -(6848 / 21)
P7 = tf.multiply(
tf.add((77096675 / 254016),
tf.add(((378515 / 1512) * eta), (-74045 / 756) * etasq)),
(np.pi))
#Produce full PN terms
PN0 = tf.multiply(P0, v0)
PN2 = tf.multiply(P2, v2)
PN3 = tf.multiply(P3, v3)
PN4 = tf.multiply(P4, v4)
PN5 = tf.multiply(tf.add(P5, tf.multiply(Pl5, v5log)), v5)
PN6 = tf.multiply(tf.add(P6, tf.multiply(Pl6, v6log)), v6)
PN7 = tf.multiply(P7, v7)
#phases = PN_cnst + PN0(1+PN1*v+PN2*v2+PN3*v3+PN4+PN5+PN6+PN7)
if phase_order == 7:
phases = tf.multiply(PN0, (1 + PN2 + PN3 + PN4 + PN5 + PN6 + PN7))
elif phase_order == 6:
phases = tf.multiply(PN0, (1 + PN2 + PN3 + PN4 + PN5 + PN6))
elif phase_order == 5:
phases = tf.multiply(PN0, (1 + PN2 + PN3 + PN4 + PN5))
elif phase_order == 4:
phases = tf.multiply(PN0, (1 + PN2 + PN3 + PN4))
elif phase_order == 3:
phases = tf.multiply(PN0, (1 + PN2 + PN3))
elif phase_order == 2:
phases = tf.multiply(PN0, (1 + PN2))
elif phase_order == 1:
phases = PN0
else:
phases = PN0
return phases
def create_network(points,
batchIds,
features,
numInputFeatures,
batchSize,
k,
numOutCat,
isTraining,
keepProbConv,
keepProbFull,
useConvDropOut=False,
useDropOutFull=True):
############################################ Compute point hierarchy
mPointHierarchy = PointHierarchy(points, features, batchIds,
[0.1, 0.4, math.sqrt(3.0) + 0.1],
"MCClassH_PH", batchSize)
############################################ Convolutions
mConvBuilder = ConvolutionBuilder(KDEWindow=0.25)
############################################ LOGITS 1
############################################ First level
# Convolution
convFeatures1 = mConvBuilder.create_convolution(
convName="Conv_1",
inPointHierarchy=mPointHierarchy,
inPointLevel=0,
inFeatures=features,
inNumFeatures=numInputFeatures,
outNumFeatures=k,
convRadius=0.1,
multiFeatureConv=True)
# Pooling
convFeatures1 = batch_norm_RELU_drop_out("Reduce_Pool_1_In_BN",
convFeatures1, isTraining,
useConvDropOut, keepProbConv)
convFeatures1 = conv_1x1("Reduce_Pool_1", convFeatures1, k, k * 2)
convFeatures1 = batch_norm_RELU_drop_out("Reduce_Pool_1_Out_BN",
convFeatures1, isTraining,
useConvDropOut, keepProbConv)
poolFeatures1 = mConvBuilder.create_convolution(
convName="Pool_1",
inPointHierarchy=mPointHierarchy,
inPointLevel=0,
outPointLevel=1,
inFeatures=convFeatures1,
inNumFeatures=k * 2,
convRadius=0.2,
KDEWindow=0.2)
############################################ Second level convolutions
#### Convolution
bnPoolFeatures1 = batch_norm_RELU_drop_out("Reduce_Conv_2_In_BN",
poolFeatures1, isTraining,
useConvDropOut, keepProbConv)
bnPoolFeatures1 = conv_1x1("Reduce_Conv_2", bnPoolFeatures1, k * 2, k * 2)
bnPoolFeatures1 = batch_norm_RELU_drop_out("Reduce_Conv_2_Out_BN",
bnPoolFeatures1, isTraining,
useConvDropOut, keepProbConv)
convFeatures2 = mConvBuilder.create_convolution(
convName="Conv_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=1,
inFeatures=bnPoolFeatures1,
inNumFeatures=k * 2,
convRadius=0.4)
convFeatures2 = tf.concat([poolFeatures1, convFeatures2], 1)
# Pooling
convFeatures2 = batch_norm_RELU_drop_out("Reduce_Pool_2_In_BN",
convFeatures2, isTraining,
useConvDropOut, keepProbConv)
convFeatures2 = conv_1x1("Reduce_Pool_2", convFeatures2, k * 4, k * 8)
convFeatures2 = batch_norm_RELU_drop_out("Reduce_Pool_2_Out_BN",
convFeatures2, isTraining,
useConvDropOut, keepProbConv)
poolFeatures2 = mConvBuilder.create_convolution(
convName="Pool_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=1,
outPointLevel=2,
inFeatures=convFeatures2,
inNumFeatures=k * 8,
convRadius=0.8,
KDEWindow=0.2)
############################################ Third level convolutions
# Convolution
bnPoolFeatures2 = batch_norm_RELU_drop_out("Reduce_Conv_3_In_BN",
poolFeatures2, isTraining,
useConvDropOut, keepProbConv)
bnPoolFeatures2 = conv_1x1("Reduce_Conv_3", bnPoolFeatures2, k * 8, k * 8)
bnPoolFeatures2 = batch_norm_RELU_drop_out("Reduce_Conv_3_Out_BN",
bnPoolFeatures2, isTraining,
useConvDropOut, keepProbConv)
convFeatures3 = mConvBuilder.create_convolution(
convName="Conv_3",
inPointHierarchy=mPointHierarchy,
inPointLevel=2,
inFeatures=bnPoolFeatures2,
inNumFeatures=k * 8,
convRadius=1.2)
convFeatures3 = tf.concat([poolFeatures2, convFeatures3], 1)
# Pooling
convFeatures3 = batch_norm_RELU_drop_out("Reduce_Pool_3_In_BN",
convFeatures3, isTraining,
useConvDropOut, keepProbConv)
convFeatures3 = conv_1x1("Reduce_Pool_3", convFeatures3, k * 16, k * 32)
convFeatures3 = batch_norm_RELU_drop_out("Reduce_Pool_3_Out_BN",
convFeatures3, isTraining,
useConvDropOut, keepProbConv)
poolFeatures3 = mConvBuilder.create_convolution(
convName="Pool_3",
inPointHierarchy=mPointHierarchy,
inPointLevel=2,
outPointLevel=3,
inFeatures=convFeatures3,
inNumFeatures=k * 32,
convRadius=math.sqrt(3.0) + 0.1,
KDEWindow=0.2)
#Fully connected MLP - Global features.
finalInput = batch_norm_RELU_drop_out("BNRELUDROP_final", poolFeatures3,
isTraining, useConvDropOut,
keepProbConv)
finalLogits1 = MLP_2_hidden(finalInput, k * 32, k * 16, k * 8, numOutCat,
"Final_Logits", keepProbFull, isTraining,
useDropOutFull)
############################################ LOGITS 2
############################################ Second level convolutions
#### Convolution
convFeatures22 = mConvBuilder.create_convolution(
convName="Conv_2_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=1,
inFeatures=mPointHierarchy.features_[1],
inNumFeatures=numInputFeatures,
outNumFeatures=k * 2,
convRadius=0.4,
multiFeatureConv=True)
# Pooling
convFeatures22 = batch_norm_RELU_drop_out("Reduce_Pool_2_2_In_BN",
convFeatures22, isTraining,
useConvDropOut, keepProbConv)
convFeatures22 = conv_1x1("Reduce_Pool_2_2", convFeatures22, k * 2, k * 8)
convFeatures22 = batch_norm_RELU_drop_out("Reduce_Pool_2_2_Out_BN",
convFeatures22, isTraining,
useConvDropOut, keepProbConv)
poolFeatures22 = mConvBuilder.create_convolution(
convName="Pool_2_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=1,
outPointLevel=2,
inFeatures=convFeatures22,
inNumFeatures=k * 8,
convRadius=0.8,
KDEWindow=0.2)
############################################ Third level convolutions
# Convolution
bnPoolFeatures22 = batch_norm_RELU_drop_out("Reduce_Conv_3_2_In_BN",
poolFeatures22, isTraining,
useConvDropOut, keepProbConv)
bnPoolFeatures22 = conv_1x1("Reduce_Conv_3_2", bnPoolFeatures22, k * 8,
k * 8)
bnPoolFeatures22 = batch_norm_RELU_drop_out("Reduce_Conv_3_2_Out_BN",
bnPoolFeatures22, isTraining,
useConvDropOut, keepProbConv)
convFeatures32 = mConvBuilder.create_convolution(
convName="Conv_3_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=2,
inFeatures=bnPoolFeatures22,
inNumFeatures=k * 8,
convRadius=1.2)
convFeatures32 = tf.concat([poolFeatures22, convFeatures32], 1)
# Pooling
convFeatures32 = batch_norm_RELU_drop_out("Reduce_Pool_3_2_In_BN",
convFeatures32, isTraining,
useConvDropOut, keepProbConv)
convFeatures32 = conv_1x1("Reduce_Pool_3_2", convFeatures32, k * 16,
k * 32)
convFeatures32 = batch_norm_RELU_drop_out("Reduce_Pool_3_2_Out_BN",
convFeatures32, isTraining,
useConvDropOut, keepProbConv)
poolFeatures32 = mConvBuilder.create_convolution(
convName="Pool_3_2",
inPointHierarchy=mPointHierarchy,
inPointLevel=2,
outPointLevel=3,
inFeatures=convFeatures32,
inNumFeatures=k * 32,
convRadius=math.sqrt(3.0) + 0.1,
KDEWindow=0.2)
#Fully connected MLP - Global features.
finalInput2 = batch_norm_RELU_drop_out("2BNRELUDROP_final", poolFeatures32,
isTraining, useConvDropOut,
keepProbConv)
finalLogits2 = MLP_2_hidden(finalInput2, k * 32, k * 16, k * 8, numOutCat,
"2Final_Logits", keepProbFull, isTraining,
useDropOutFull)
############################################ PATH DROPOUT
counter = tf.constant(0.0, dtype=tf.float32)
probability = tf.random_uniform([1])
mask1 = tf.less_equal(probability[0], tf.constant(0.66))
mask1 = tf.maximum(tf.cast(mask1, tf.float32),
tf.cast(tf.logical_not(isTraining), tf.float32))
counter = tf.add(counter, mask1)
finalLogits1 = tf.scalar_mul(mask1, finalLogits1)
mask2 = tf.greater_equal(probability[0], tf.constant(0.33))
mask2 = tf.maximum(tf.cast(mask2, tf.float32),
tf.cast(tf.logical_not(isTraining), tf.float32))
counter = tf.add(counter, mask2)
finalLogits2 = tf.scalar_mul(mask2, finalLogits2)
counter = tf.multiply(tf.constant((2.0), dtype=tf.float32),
tf.reciprocal(counter))
return tf.scalar_mul(counter, tf.add(finalLogits1, finalLogits2))
def custom_conv2d_pos_for_assignment(x,
adj,
out_channels,
M,
biasMask=True,
translation_invariance=False,
rotation_invariance=False):
batch_size, input_size, in_channels_ass = x.get_shape().as_list()
in_channels_weights = in_channels_ass - 3
#in_channels_ass = 6
xn = tf.slice(x, [0, 0, 0],
[-1, -1, in_channels_weights]) # take normals only
W0 = weight_variable([M, out_channels, in_channels_weights])
b = bias_variable([out_channels])
u = assignment_variable([M, in_channels_ass])
c = assignment_variable([M])
adj_batch_size, input_size, K = adj.get_shape().as_list()
# Calculate neighbourhood size for each input - [batch_size, input_size, neighbours]
adj_size = tf.count_nonzero(adj, 2)
# [ N, K] ?
#deal with unconnected points: replace NaN with 0
non_zeros = tf.not_equal(adj_size, 0)
adj_size = tf.cast(adj_size, tf.float32)
adj_size_inv = tf.where(non_zeros, tf.reciprocal(adj_size),
tf.zeros_like(adj_size))
# [batch_size, input_size, 1, 1]
#adj_size = tf.reshape(adj_size, [batch_size, input_size, 1, 1])
# adj_size = tf.reshape(adj_size, [batch_size, -1, 1, 1])
adj_size_inv = tf.reshape(adj_size_inv, [adj_batch_size, -1, 1, 1])
if (translation_invariance == False) and (rotation_invariance == False):
vn = assignment_variable([M, in_channels_weights])
elif translation_invariance == True:
print("Translation-invariant\n")
un = tf.slice(u, [0, 0], [-1, in_channels_weights])
vn = -un
# # [batch_size, input_size, K, M]
# q = get_weight_assigments_translation_invariance(x, adj, u, c)
# elif rotation_invariance == True:
# print("Rotation-invariant\n")
# # [batch_size, input_size, K, M]
# if in_channels==3:
# q = get_weight_assigments_rotation_invariance(x, adj, u, c)
# elif in_channels==4:
# q = get_weight_assigments_rotation_invariance_with_area(x, adj, u, c)
# elif in_channels==6:
# q = get_weight_assigments_rotation_invariance_with_position(x, adj, u, c)
# Make new assignement, that is translation invariant wrt position, but not normals
# vn = assignment_variable([M, in_channels_weights])
up = tf.slice(u, [0, in_channels_weights], [-1, -1])
vp = -up
v = tf.concat([vn, vp], axis=-1)
# [batch_size, in_channels, input_size]
x = tf.transpose(x, [0, 2, 1])
xn = tf.transpose(xn, [0, 2, 1])
W = tf.reshape(W0, [M * out_channels, in_channels_weights])
# Multiple w and x -> [batch_size, M*out_channels, input_size]
wx = tf.map_fn(lambda x: tf.matmul(W, x), xn)
# Reshape and transpose wx into [batch_size, input_size, M*out_channels]
wx = tf.transpose(wx, [0, 2, 1])
# Get patches from wx - [batch_size, input_size, K(neighbours-here input_size), M*out_channels]
patches = get_patches(wx, adj)
# [batch_size, input_size, K, M]
q = get_weight_assigments(x, adj, u, v, c)
# Element wise multiplication of q and patches for each input -- [batch_size, input_size, K, M, out]
#patches = tf.reshape(patches, [batch_size, input_size, K, M, out_channels])
patches = tf.reshape(patches, [batch_size, -1, K, M, out_channels])
# [out, batch_size, input_size, K, M]
patches = tf.transpose(patches, [4, 0, 1, 2, 3])
print("\npatches shape = " + str(patches.shape))
patches = tf.multiply(q, patches)
print("patches shape = " + str(patches.shape))
patches = tf.transpose(patches, [1, 2, 3, 4, 0])
print("patches shape = " + str(patches.shape))
# Add all the elements for all neighbours for a particular m sum_{j in N_i} qwx -- [batch_size, input_size, M, out]
patches = tf.reduce_sum(patches, axis=2)
patches = tf.multiply(adj_size_inv, patches)
# Add add elements for all m
patches = tf.reduce_sum(patches, axis=2)
# [batch_size, input_size, out]
if biasMask:
# NEW!! Add bias only to nodes with at least one active node in neighbourhood
patches = tf.where(
tf.tile(tf.expand_dims(non_zeros, axis=-1), [1, 1, out_channels]),
patches + b, patches)
else:
patches = patches + b
return patches, W0
def build_loss(data):
t0 = datetime.datetime.now()
print('Initializing Graph Tensors...')
print('Time Now:',time.strftime('%H:%M:%S',time.localtime(time.time())))
node = [Node(dnode) for dnode in data['dnode']]
od = [OD(dod) for dod in data['dod']]
path = [Path(dpath) for dpath in data['dpath']]
link = [Link(dlink) for dlink in data['dlink']]
t1 = datetime.datetime.now()
print('Using Time: ',t1-t0,'\n')
print('Building Connections from Node to OD...')
print('Time Now:',time.strftime('%H:%M:%S',time.localtime(time.time())))
# node layer to od layer
## NOTE: simplify due to [each zone only include one node]
source_zones_ids = set([od_w.from_zone_id for od_w in od])
source_zones = [node_i for node_i in node if node_i.node_id in source_zones_ids]
source_zones = sorted(source_zones, key=lambda node_i: node_i.node_id)
for od_w in od:
node_i = node[int(od_w.from_zone_id)]
od_w.q_ = tf.multiply(od_w.gamma_norm_, node_i.alpha_, name='OD/q/OD_'+str(od_w.od_id))
t2 = datetime.datetime.now()
print('Using Time: ',t2-t1,'\n')
print('Building Connections from OD to Path...')
print('Time Now:',time.strftime('%H:%M:%S',time.localtime(time.time())))
# od layer to path layer
for path_r in path:
for od_w in od:
if od_w.from_zone_id == path_r.from_zone_id and od_w.to_zone_id == path_r.to_zone_id:
path_r.belonged_od = od_w
od_w.including_paths.append(path_r)
# TODO1: logit model
for path_r in path:
## NOTE: add cost here if needed
path_r.exp_ = tf.exp(path_r.belonged_od.theta_ * path_r.t)
for od_w in od:
od_w.exp_reci_ = tf.reciprocal(tf.reduce_sum(tf.stack([path_r.exp_ for path_r in od_w.including_paths])))
for path_r in path:
path_r.rou_ = tf.multiply(path_r.exp_, path_r.belonged_od.exp_reci_, name='Path/rou/Path_'+str(path_r.path_id))
# TODO1: logit model
for path_r in path:
path_r.flow_ = tf.multiply(path_r.rou_, path_r.belonged_od.q_, name='Path/flow/Path_'+str(path_r.path_id))
t3 = datetime.datetime.now()
print('Using Time: ',t3-t2,'\n')
print('Building Connections from Path to Link...')
print('Time Now:',time.strftime('%H:%M:%S',time.localtime(time.time())))
# path layer to link layer
link_dict = dict()
for link_l in link:
link_dict[(link_l.from_node_id, link_l.to_node_id)] = link_l
for path_r in path:
node_list = path_r.node_sequence.split(';')
## TODO: check whether node_list is longer than 2
for i in range(1,len(node_list)):
link_l = link_dict[(int(node_list[i-1]),int(node_list[i]))]
link_l.belonged_paths.append(path_r)
for link_l in link:
link_l.v_ = tf.reduce_sum(tf.stack([path_r.flow_ for path_r in link_l.belonged_paths],
name='Link/v_vector/link_'+str(link_l.link_id)),
name='Link/v/link_'+str(link_l.link_id))
t4 = datetime.datetime.now()
print('Using Time: ',t4-t3,'\n')
print('Building Loss...')
print('Time Now:',time.strftime('%H:%M:%S',time.localtime(time.time())))
# build loss and optimize
## TODO: confirm the first term of loss, whether only consider source zones (people go out)
f1 = tf.reduce_sum(tf.stack([tf.pow(tf.subtract(tf.truediv(node_i.alpha_, node_i.PopTR,
name='loss/alpha/div'),
1, name='loss/alpha/sub'),
2, name='loss/alpha/term')
for node_i in source_zones], name='loss/alpha/vector'),
name='loss/alpha/alpha')
tf.summary.scalar('loss/alpha/alpha', f1) ####
f2 = tf.reduce_sum(tf.stack([tf.pow(tf.subtract(tf.truediv(od_w.gamma_norm_, od_w.split,
name='loss/gamma/div'),
1, name='loss/gamma/sub'),
2, name='loss/gamma/term')
for od_w in od], name='loss/gamma/vector'),
name='loss/gamma/gamma')
tf.summary.scalar('loss/gamma/gamma', f2) ####
f3 = tf.reduce_sum(tf.stack([tf.pow(tf.subtract(tf.truediv(link_l.v_, link_l.count,
name='loss/flow/div'),
1, name='loss/flow/sub'),
2, name='loss/flow/term')
for link_l in link if link_l.is_observed], name='loss/flow/vector'),
name='loss/flow/flow')
tf.summary.scalar('loss/flow/flow', f3) ####
loss = tf.reduce_sum(tf.stack([f1, f2, f3], name='loss/total_loss_vector'), name='loss/total_loss')
tf.summary.scalar('loss/total_loss_vector', loss) ####
t5 = datetime.datetime.now()
print('Using Time: ',t5-t4,'\n')
print('Adding moniter...')
alpha_ = tf.stack([node_i.alpha_ for node_i in node], name='Node/alpha/vector')
tf.summary.histogram('Node/alpha/vector', alpha_) ####
gamma_ = tf.stack([od_w.gamma_norm_ for od_w in od], name='OD/gamma/vector')
tf.summary.histogram('OD/gamma/vector', gamma_) ####
q_ = tf.stack([od_w.q_ for od_w in od], name='OD/q/vector')
tf.summary.histogram('OD/q/vector', q_) ####
rou_ = tf.stack([path_r.rou_ for path_r in path], name='Path/rou/vector')
tf.summary.histogram('Path/rou/vector', rou_) ####
flow_ = tf.stack([path_r.flow_ for path_r in path], name='Path/flow/vector')
tf.summary.histogram('Path/flow/vector', flow_) ####
v_ = tf.stack([link_l.v_ for link_l in link], name='Link/v/vector')
tf.summary.histogram('Link/v/vector', v_ ) ####
v_ob_ = tf.stack([link_l.v_ for link_l in link if link_l.is_observed], name='Link/v_observed/vector')
tf.summary.histogram('Link/v_observed/vector', v_ob_ )
t6 = datetime.datetime.now()
print('Using Time: ',t6-t5,'\n')
return f1, f2, f3, loss
def _compute_loss(self, logits):
"""Compute optimization loss."""
target_output = self.iterator.target_output
if self.time_major:
target_output = tf.transpose(target_output)
max_time = self.get_max_time(target_output)
### experiments
# batch_cd_list = tf.zeros([1, tf.cast(logits.get_shape()[2], tf.int32)])
# i = tf.constant(0)
# while_condition = lambda i, b: tf.less(i, self.batch_size)
# def body(i, batch_cd_list):
# one_hot = tf.one_hot(target_output[i], logits.get_shape()[2])
# dist = tf.reduce_sum(one_hot, axis=0)
# cd_list = tf.divide(dist, tf.reduce_sum(dist))
# j = tf.constant(1)
# def while_condition_2(j, dist, cd_list):
# return tf.less(j, max_time)
# def body_2(j, dist, cd_list):
# dist = tf.subtract(dist, tf.one_hot(tf.argmax(logits[i][j-1]),
# logits.get_shape()[2]))
# cd_list = tf.concat([cd_list, tf.divide(dist, tf.reduce_sum(dist))], axis=0)
# tf.Print(j, [j], message="This is j!!!!")
# return [tf.add(j, 1), dist, cd_list]
# s = tf.while_loop(while_condition_2, body_2, [j, dist, cd_list],
# shape_invariants=[j.get_shape(),
# dist.get_shape(), tf.TensorShape([None])])
# cd_list = tf.reshape(cd_list, [-1, tf.cast(logits.get_shape()[2], tf.int32)])
# def assign1(batch_cd_list, cd_list):
# batch_cd_list = cd_list
# print("this is where I am" + str(cd_list.get_shape()))
# return batch_cd_list
# def assign2(batch_cd_list, cd_list):
# print(batch_cd_list.get_shape())
# batch_cd_list = tf.concat([batch_cd_list, cd_list], axis=0)
# print("I am here as well")
# return batch_cd_list
# batch_cd_list = tf.cond(tf.equal(i, tf.constant(0)), lambda: assign1(batch_cd_list, cd_list), lambda: assign2(batch_cd_list, cd_list))
# print("see this: ", batch_cd_list)
# return [tf.add(i, 1), batch_cd_list]
# r = tf.while_loop(while_condition, body, [i, batch_cd_list],
# shape_invariants=[i.get_shape(), tf.TensorShape([None, logits.get_shape()[2]])])
# batch_cd = tf.reshape(batch_cd_list, [self.batch_size, max_time, tf.cast(logits.get_shape()[2], tf.int32)])
# """
# one_hot_target_output = tf.one_hot(target_output[0][0], logits.get_shape()[2])
# """
preds = tf.argmax(logits, axis=2)
one_hot_preds = tf.one_hot(preds, logits.get_shape()[2])
cum_preds = tf.cumsum(one_hot_preds, axis=0)
cum_preds = tf.slice(cum_preds, [0, 0, 0], [
max_time - 1, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32)
])
cum_preds = tf.concat([
tf.zeros(
[1, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32)]), cum_preds
],
axis=0)
# Now, we have the cumulative predictions tensor.
one_hot_targets = tf.one_hot(target_output, logits.get_shape()[2])
targets_test = tf.reshape(one_hot_targets, [
max_time, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32)
])
dist = tf.reduce_sum(one_hot_targets, axis=0, keep_dims=True)
rep_dist = tf.tile(dist, [max_time, 1, 1])
target_dist = tf.subtract(rep_dist, cum_preds)
target_dist = tf.maximum(
target_dist,
tf.zeros([
max_time, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32)
]))
# setting eos count to 1
target_dist_first = tf.slice(target_dist, [0, 0, 0],
[max_time, self.batch_size, 2])
target_dist_last = tf.slice(target_dist, [0, 0, 3], [
max_time, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32) - 3
])
target_dist = tf.concat(
[target_dist_first,
tf.ones([max_time, self.batch_size, 1])],
axis=2)
target_dist = tf.concat([target_dist, target_dist_last], axis=2)
# eos count has been set to 1
# Pushing in Puru's method for precision at k.
# Loss at false negatives = -log(p)
# Loss at false positives = -log(1-p)
k = 3
beta = 0.1
top_k_values, top_k_targets = tf.nn.top_k(target_dist, k=k)
top_k_predicted_values, top_k_predicted = tf.nn.top_k(logits, k=k)
false_negatives = tf.sets.set_difference(top_k_targets,
top_k_predicted)
false_positives = tf.sets.set_difference(top_k_predicted,
top_k_targets)
dense_fn = tf.sparse_tensor_to_dense(false_negatives)
one_hot_fn = tf.one_hot(indices=dense_fn,
depth=logits.get_shape()[2],
on_value=1.0)
target_k = tf.reduce_sum(one_hot_fn, axis=2)
print("target_k vector shape: ", target_k.get_shape())
dense_fp = tf.sparse_tensor_to_dense(false_positives)
one_hot_fp = tf.one_hot(indices=dense_fp,
depth=logits.get_shape()[2],
on_value=1.0)
predicted_k = tf.reduce_sum(one_hot_fp, axis=2)
# normalising the target distribution.
# Have tried softmax normalization and linear normalization.
target_sums = tf.reduce_sum(target_dist, axis=2, keep_dims=True)
reci_target_sums = tf.reciprocal(target_sums)
reci_target_sums_rep = tf.tile(
reci_target_sums,
[1, 1, tf.cast(logits.get_shape()[2], tf.int32)])
target_dist_norm = tf.multiply(target_dist, reci_target_sums_rep)
# target_dist_norm = tf.nn.softmax(target_dist)
one_minus_logits = tf.subtract(
tf.ones([
max_time, self.batch_size,
tf.cast(logits.get_shape()[2], tf.int32)
]), logits)
crossent_fn = tf.nn.softmax_cross_entropy_with_logits(labels=target_k,
logits=logits)
crossent_fp = tf.nn.softmax_cross_entropy_with_logits(
labels=predicted_k, logits=one_minus_logits)
# crossent_impl = tf.nn.softmax_cross_entropy_with_logits(
# labels=target_dist_norm, logits=logits)
crossent_impl = (crossent_fn + crossent_fp) / 2.0
####
# target_output_test = tf.reshape(target_output, [self.batch_size, max_time])
# print("reshape successful!!!!", target_output.get_shape())
crossent_orig = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=target_output, logits=logits)
target_weights = tf.sequence_mask(self.iterator.target_sequence_length,
max_time,
dtype=logits.dtype)
if self.time_major:
target_weights = tf.transpose(target_weights)
# cross entropy is a weighted combiation of original cross entropy and
# the one implemented by us (crossent_impl)
crossent = (1.0 - beta) * crossent_orig + beta * crossent_impl
loss = tf.reduce_sum(crossent * target_weights) / tf.to_float(
self.batch_size)
return loss
def custom_conv2d_only_pos_for_assignment(x,
adj,
out_channels,
M,
translation_invariance=False,
rotation_invariance=False):
batch_size, input_size, in_channels_ass = x.get_shape().as_list()
in_channels_weights = in_channels_ass - 3
#in_channels_ass = 6
xn = tf.slice(x, [0, 0, 0],
[-1, -1, in_channels_weights]) # take normals only
xp = tf.slice(x, [0, 0, in_channels_weights], [-1, -1, -1])
W0 = weight_variable([M, out_channels, in_channels_weights])
b = bias_variable([out_channels])
u = assignment_variable([M, 3])
c = assignment_variable([M])
if translation_invariance:
q = get_weight_assigments_translation_invariance(xp, adj, u, c)
else:
v = assignment_variable([M, 3])
q = get_weight_assigments(tf.transpose(
xp,
[0, 2, 1],
), adj, u, v, c)
adj_batch_size, input_size, K = adj.get_shape().as_list()
# Calculate neighbourhood size for each input - [batch_size, input_size, neighbours]
adj_size = tf.count_nonzero(adj, 2)
# [ N, K] ?
#deal with unconnected points: replace NaN with 0
non_zeros = tf.not_equal(adj_size, 0)
adj_size = tf.cast(adj_size, tf.float32)
adj_size_inv = tf.where(non_zeros, tf.reciprocal(adj_size),
tf.zeros_like(adj_size))
# [batch_size, input_size, 1, 1]
#adj_size = tf.reshape(adj_size, [batch_size, input_size, 1, 1])
# adj_size = tf.reshape(adj_size, [batch_size, -1, 1, 1])
adj_size_inv = tf.reshape(adj_size_inv, [adj_batch_size, -1, 1, 1])
# [batch_size, in_channels, input_size]
x = tf.transpose(x, [0, 2, 1])
xn = tf.transpose(xn, [0, 2, 1])
W = tf.reshape(W0, [M * out_channels, in_channels_weights])
# Multiple w and x -> [batch_size, M*out_channels, input_size]
wx = tf.map_fn(lambda x: tf.matmul(W, x), xn)
# Reshape and transpose wx into [batch_size, input_size, M*out_channels]
wx = tf.transpose(wx, [0, 2, 1])
# Get patches from wx - [batch_size, input_size, K(neighbours-here input_size), M*out_channels]
patches = get_patches(wx, adj)
# [batch_size, input_size, K, M]
# Element wise multiplication of q and patches for each input -- [batch_size, input_size, K, M, out]
#patches = tf.reshape(patches, [batch_size, input_size, K, M, out_channels])
patches = tf.reshape(patches, [batch_size, -1, K, M, out_channels])
# [out, batch_size, input_size, K, M]
patches = tf.transpose(patches, [4, 0, 1, 2, 3])
print("\npatches shape = " + str(patches.shape))
patches = tf.multiply(q, patches)
print("patches shape = " + str(patches.shape))
patches = tf.transpose(patches, [1, 2, 3, 4, 0])
print("patches shape = " + str(patches.shape))
# Add all the elements for all neighbours for a particular m sum_{j in N_i} qwx -- [batch_size, input_size, M, out]
patches = tf.reduce_sum(patches, axis=2)
patches = tf.multiply(adj_size_inv, patches)
# Add add elements for all m
patches = tf.reduce_sum(patches, axis=2)
# [batch_size, input_size, out]
patches = patches + b
return patches, W0
def _resample_inv_dst_weighting(self, inputs, sample_coords):
in_size = inputs.shape.as_list()
in_spatial_size = in_size[1:-1]
in_spatial_rank = infer_spatial_rank(inputs)
out_rank = len(sample_coords.shape.as_list())
self.N = 2 ** in_spatial_rank
binary_neighbour_ids = [
[int(c) for c in format(i, '0%ib' % in_spatial_rank)]
for i in range(self.N)]
weight_id = [[[c, i] for i, c in enumerate(bc)]
for bc in binary_neighbour_ids]
sample_coords = tf.transpose(
sample_coords, [out_rank - 1, 0] + list(range(1, out_rank - 1)))
# broadcasting input spatial size for boundary functions
b_size = tf.reshape(in_spatial_size,
[len(in_spatial_size)] + [1] * (out_rank - 1))
# find floor and ceil coordinates
all_coords_f = tf.stack([
self.boundary_func(tf.floor(sample_coords), b_size),
self.boundary_func(tf.ceil(sample_coords), b_size)])
# find N weights associated to each output point
diff = tf.stack(
[tf.squared_difference(sample_coords - EPS, all_coords_f[0]),
tf.squared_difference(sample_coords + EPS, all_coords_f[1])])
# gather_nd for both matrices, the same as:
# point_weights = tf.gather_nd(diff, weight_id)
# knots_id = tf.gather_nd(all_coords_f, weight_id)
n_val = tf.gather_nd(tf.stack([diff, all_coords_f], axis=-1), weight_id)
n_val = tf.unstack(n_val, axis=-1)
point_weights, knots_id = n_val[0], n_val[1]
# inverse distance weighting
# sum_i (w_i*p_i/(sum_j w_j)) w_i = 1/((p-p_i)^2)
# point_weights shape:
# `[N, input_rank, b, sp_dim_0, ..., sp_dim_K]`
# where:
# `N` is 2**source data spatial rank
# `b` is batch size,
# `sp_dim_0` is the output spatial output 0,
#
# `point_weights` represents (p - p_i)^2
# with i= 0...2**source_rank neighbours
# (to do: these operations could be refactored as a resampling kernel)
point_weights = tf.reduce_sum(point_weights, axis=1)
# skip this as power = 2.0:
# self.power = 1.0
# point_weights = tf.pow(point_weights, self.power / 2.0)
point_weights = tf.reciprocal(point_weights)
point_weights = point_weights / tf.reduce_sum(point_weights, axis=0)
# find N neighbours associated to each output point
knots_id = tf.transpose(tf.cast(knots_id, COORDINATES_TYPE),
[0] + list(range(2, out_rank + 1)) + [1])
# get values of N neighbours
samples = [
tf.gather_nd(img, knots) for (img, knots) in
zip(tf.unstack(inputs, axis=0), tf.unstack(knots_id, axis=1))]
samples = tf.stack(samples, axis=1)
# weighted average over N neighbours
return tf.reduce_sum(
samples * tf.expand_dims(point_weights, axis=-1), axis=0)
def test_reciprocal(self):
x_val = np.array([1.0, 2.0, -3.0, -4.0], dtype=np.float32).reshape((2, 2))
x = tf.placeholder(tf.float32, [2, 2], name=_TFINPUT)
x_ = tf.reciprocal(x)
_ = tf.identity(x_, name=_TFOUTPUT)
self._run_test_case([_OUTPUT], {_INPUT: x_val}, rtol=1e-04)
def _tf_normal(self, y, mu, sigma):
oneDivSqrtTwoPI = 1 / math.sqrt(2 * math.pi) # normalisation factor for gaussian, not needed.
result = tf.subtract(y, mu)
result = tf.multiply(result, tf.reciprocal(sigma))
result = -tf.square(result) / 2
return tf.multiply(tf.exp(result), tf.reciprocal(sigma)) * oneDivSqrtTwoPI
def forward(self, inputs, grid, is_training=True, reuse=False):
def preprocessing(inputs):
dims = inputs.get_shape()
if len(dims) == 3:
inputs = tf.expand_dims(inputs, dim=0)
mean_BGR = tf.reshape(self.mean_BGR, [1, 1, 1, 3])
inputs = inputs[:, :, :, ::-1] + mean_BGR
return inputs
## -----------------------depth and normal FCN--------------------------
inputs = preprocessing(inputs)
with slim.arg_scope(
[slim.conv2d, slim.conv2d_transpose],
activation_fn=tf.nn.relu,
stride=1,
padding='SAME',
weights_initializer=weight_from_caffe(self.pretrain_weight),
biases_initializer=bias_from_caffe(self.pretrain_weight)):
with tf.variable_scope('fcn', reuse=reuse):
##---------------------vgg depth------------------------------------
conv1 = slim.repeat(inputs,
2,
slim.conv2d,
64, [3, 3],
scope='conv1')
pool1 = slim.max_pool2d(conv1, [3, 3],
stride=2,
padding='SAME',
scope='pool1')
conv2 = slim.repeat(pool1,
2,
slim.conv2d,
128, [3, 3],
scope='conv2')
pool2 = slim.max_pool2d(conv2, [3, 3],
stride=2,
padding='SAME',
scope='pool2')
conv3 = slim.repeat(pool2,
3,
slim.conv2d,
256, [3, 3],
scope='conv3')
pool3 = slim.max_pool2d(conv3, [3, 3],
stride=2,
padding='SAME',
scope='pool3')
conv4 = slim.repeat(pool3,
3,
slim.conv2d,
512, [3, 3],
scope='conv4')
pool4 = slim.max_pool2d(conv4, [3, 3],
stride=1,
padding='SAME',
scope='pool4')
conv5 = slim.repeat(pool4,
3,
slim.conv2d,
512, [3, 3],
rate=2,
scope='conv5')
pool5 = slim.max_pool2d(conv5, [3, 3],
stride=1,
padding='SAME',
scope='pool5')
pool5a = slim.avg_pool2d(pool5, [3, 3],
stride=1,
padding='SAME',
scope='pool5a')
fc6 = slim.conv2d(pool5a,
1024, [3, 3],
stride=1,
rate=12,
scope='fc6')
fc6 = slim.dropout(fc6,
0.5,
is_training=is_training,
scope='drop6')
fc7 = slim.conv2d(fc6, 1024, [1, 1], scope='fc7')
fc7 = slim.dropout(fc7,
0.5,
is_training=is_training,
scope='drop7')
pool6_1x1 = slim.avg_pool2d(fc7, [61, 81],
stride=[61, 81],
padding='SAME',
scope='pool6_1x1')
pool6_1x1_norm = slim.unit_norm(pool6_1x1,
dim=3,
scope='pool6_1x1_norm_new')
pool6_1x1_norm_scale = pool6_1x1_norm * 10
pool6_1x1_norm_upsample = tf.tile(
pool6_1x1_norm_scale, [1, 61, 81, 1],
name='pool6_1x1_norm_upsample')
out = tf.concat([fc7, pool6_1x1_norm_upsample],
axis=-1,
name='out')
out_reduce = slim.conv2d(out,
256, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='out_reduce',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(
self.pretrain_weight))
out_conv = slim.conv2d(out_reduce,
256, [3, 3],
activation_fn=tf.nn.relu,
stride=1,
scope='out_conv',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(
self.pretrain_weight))
out_conv_increase = slim.conv2d(
out_conv,
1024, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='out_conv_increase',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(self.pretrain_weight))
fc8_nyu_depth = slim.conv2d(out_conv_increase,
1, [1, 1],
activation_fn=None,
scope='fc8_nyu_depth')
fc8_upsample = tf.image.resize_images(
fc8_nyu_depth, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
#---------------------------------------vgg depth end ---------------------------------------
## ----------------- vgg norm---------------------------------------------------------------
conv1_norm = slim.repeat(inputs,
2,
slim.conv2d,
64, [3, 3],
scope='conv1_norm')
pool1_norm = slim.max_pool2d(conv1_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool1_norm')
conv2_norm = slim.repeat(pool1_norm,
2,
slim.conv2d,
128, [3, 3],
scope='conv2_norm')
pool2_norm = slim.max_pool2d(conv2_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool2_norm')
conv3_norm = slim.repeat(pool2_norm,
3,
slim.conv2d,
256, [3, 3],
scope='conv3_norm')
pool3_norm = slim.max_pool2d(conv3_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool3_norm')
conv4_norm = slim.repeat(pool3_norm,
3,
slim.conv2d,
512, [3, 3],
scope='conv4_norm')
pool4_norm = slim.max_pool2d(conv4_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool4_norm')
conv5_norm = slim.repeat(pool4_norm,
3,
slim.conv2d,
512, [3, 3],
rate=2,
scope='conv5_norm')
pool5_norm = slim.max_pool2d(conv5_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool5_norm')
pool5a_norm = slim.avg_pool2d(pool5_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool5a_norm')
fc6_norm = slim.conv2d(pool5a_norm,
1024, [3, 3],
stride=1,
rate=12,
scope='fc6_norm')
fc6_norm = slim.dropout(fc6_norm,
0.5,
is_training=is_training,
scope='drop6_norm')
fc7_norm = slim.conv2d(fc6_norm,
1024, [1, 1],
scope='fc7_norm')
fc7_norm = slim.dropout(fc7_norm,
0.5,
is_training=is_training,
scope='drop7_norm')
pool6_1x1_norm_new = slim.avg_pool2d(
fc7_norm, [61, 81],
stride=[61, 81],
padding='SAME',
scope='pool6_1x1_norm_new')
pool6_1x1_norm_norm = slim.unit_norm(
pool6_1x1_norm_new, dim=3, scope='pool6_1x1_norm_new')
pool6_1x1_norm_scale_norm = pool6_1x1_norm_norm * 10
pool6_1x1_norm_upsample_norm = tf.tile(
pool6_1x1_norm_scale_norm, [1, 61, 81, 1],
name='pool6_1x1_norm_upsample')
out_norm = tf.concat([fc7_norm, pool6_1x1_norm_upsample_norm],
axis=-1,
name='out_norm')
fc8_nyu_norm_norm = slim.conv2d(out_norm,
3, [1, 1],
activation_fn=None,
scope='fc8_nyu_norm_norm')
fc8_upsample_norm = tf.image.resize_images(
fc8_nyu_norm_norm, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
fc8_upsample_norm = slim.unit_norm(fc8_upsample_norm, dim=3)
#-------------------------------------vgg norm end---------------------------------------------
# ------------- depth to normal + norm refinement---------------------------------------------------
with tf.variable_scope('noise', reuse=reuse):
fc8_upsample_norm = tf.squeeze(fc8_upsample_norm)
fc8_upsample_norm = tf.reshape(
fc8_upsample_norm,
[self.batch_size, self.crop_size_h, self.crop_size_w, 3])
norm_matrix = tf.extract_image_patches(
images=fc8_upsample_norm,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
matrix_c = tf.reshape(norm_matrix, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
fc8_upsample_norm = tf.expand_dims(fc8_upsample_norm, axis=4)
angle = tf.matmul(matrix_c, fc8_upsample_norm)
valid_condition = tf.greater(angle, self.thresh)
valid_condition_all = tf.tile(valid_condition, [1, 1, 1, 1, 3])
exp_depth = tf.exp(fc8_upsample * 0.69314718056)
depth_repeat = tf.tile(exp_depth, [1, 1, 1, 3])
points = tf.multiply(grid, depth_repeat)
point_matrix = tf.extract_image_patches(
images=points,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
matrix_a = tf.reshape(point_matrix, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
matrix_a_zero = tf.zeros_like(matrix_a, dtype=tf.float32)
matrix_a_valid = tf.where(valid_condition_all, matrix_a,
matrix_a_zero)
matrix_a_trans = tf.matrix_transpose(matrix_a_valid,
name='matrix_transpose')
matrix_b = tf.ones(shape=[
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 1
])
point_multi = tf.matmul(matrix_a_trans,
matrix_a_valid,
name='matrix_multiplication')
with tf.device('cpu:0'):
matrix_deter = tf.matrix_determinant(point_multi)
inverse_condition = tf.greater(matrix_deter, 1e-5)
inverse_condition = tf.expand_dims(inverse_condition, axis=3)
inverse_condition = tf.expand_dims(inverse_condition, axis=4)
inverse_condition_all = tf.tile(inverse_condition,
[1, 1, 1, 3, 3])
diag_constant = tf.ones([3], dtype=tf.float32)
diag_element = tf.diag(diag_constant)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_matrix = tf.tile(diag_element, [
self.batch_size, self.crop_size_h, self.crop_size_w, 1, 1
])
inversible_matrix = tf.where(inverse_condition_all,
point_multi, diag_matrix)
with tf.device('cpu:0'):
inv_matrix = tf.matrix_inverse(inversible_matrix)
generated_norm = tf.matmul(
tf.matmul(inv_matrix, matrix_a_trans), matrix_b)
norm_normalize = slim.unit_norm((generated_norm), dim=3)
norm_normalize = tf.reshape(
norm_normalize,
[self.batch_size, self.crop_size_h, self.crop_size_w, 3])
norm_scale = norm_normalize * 10.0
conv1_noise = slim.repeat(norm_scale,
2,
slim.conv2d,
64, [3, 3],
scope='conv1_noise')
pool1_noise = slim.max_pool2d(conv1_noise, [3, 3],
stride=2,
padding='SAME',
scope='pool1_noise') #
conv2_noise = slim.repeat(pool1_noise,
2,
slim.conv2d,
128, [3, 3],
scope='conv2_noise')
conv3_noise = slim.repeat(conv2_noise,
3,
slim.conv2d,
256, [3, 3],
scope='conv3_noise')
fc1_noise = slim.conv2d(conv3_noise,
512, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='fc1_noise',
padding='SAME')
encode_norm_noise = slim.conv2d(fc1_noise,
3, [3, 3],
activation_fn=None,
stride=1,
scope='encode_norm_noise',
padding='SAME')
encode_norm_upsample_noise = tf.image.resize_images(
encode_norm_noise, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
sum_norm_noise = tf.add(norm_normalize,
encode_norm_upsample_noise)
norm_pred_noise = slim.unit_norm(sum_norm_noise, dim=3)
norm_pred_all = tf.concat([
tf.expand_dims(tf.squeeze(fc8_upsample_norm), axis=0),
norm_pred_noise, inputs * 0.00392156862
],
axis=3)
norm_pred_all = slim.repeat(
norm_pred_all,
3,
slim.conv2d,
128, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_norm_noise_new')
norm_pred_all = slim.repeat(
norm_pred_all,
3,
slim.conv2d,
128, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_norm_noise_new')
norm_pred_final = slim.conv2d(
norm_pred_all,
3, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='norm_conv3_noise_new')
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
# ------------- normal to depth + depth refinement---------------------------------------------------
with tf.variable_scope('norm_depth', reuse=reuse):
grid_patch = tf.extract_image_patches(
images=grid,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
grid_patch = tf.reshape(grid_patch, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
_, _, depth_data = tf.split(value=matrix_a,
num_or_size_splits=3,
axis=4)
tmp_matrix_zero = tf.zeros_like(angle, dtype=tf.float32)
valid_angle = tf.where(valid_condition, angle, tmp_matrix_zero)
lower_matrix = tf.matmul(matrix_c, tf.expand_dims(grid,
axis=4))
condition = tf.greater(lower_matrix, 1e-5)
tmp_matrix = tf.ones_like(lower_matrix)
lower_matrix = tf.where(condition, lower_matrix, tmp_matrix)
lower = tf.reciprocal(lower_matrix)
valid_angle = tf.where(condition, valid_angle, tmp_matrix_zero)
upper = tf.reduce_sum(tf.multiply(matrix_c, grid_patch), [4])
ratio = tf.multiply(lower, tf.expand_dims(upper, axis=4))
estimate_depth = tf.multiply(ratio, depth_data)
valid_angle = tf.multiply(
valid_angle,
tf.reciprocal(
tf.tile(
tf.reduce_sum(valid_angle, [3, 4], keep_dims=True)
+ 1e-5, [1, 1, 1, 81, 1])))
depth_stage1 = tf.reduce_sum(
tf.multiply(estimate_depth, valid_angle), [3, 4])
depth_stage1 = tf.expand_dims(tf.squeeze(depth_stage1), axis=2)
depth_stage1 = tf.clip_by_value(depth_stage1, 0, 10.0)
exp_depth = tf.expand_dims(tf.squeeze(exp_depth), axis=2)
depth_all = tf.expand_dims(tf.concat([
depth_stage1, exp_depth,
tf.squeeze(inputs) * 0.00392156862
],
axis=2),
axis=0)
depth_pred_all = slim.repeat(
depth_all,
3,
slim.conv2d,
128, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_depth_noise_new')
depth_pred_all = slim.repeat(
depth_pred_all,
3,
slim.conv2d,
128, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_depth_noise_new')
final_depth = slim.conv2d(
depth_pred_all,
1, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='depth_conv3_noise_new')
with tf.variable_scope('edge_refinemet', reuse=reuse):
print(inputs.shape)
edges = tf.py_func(myfunc_canny, [inputs], tf.float32)
edges = tf.reshape(edges,
[1, self.crop_size_h, self.crop_size_w, 1])
edge_input_depth = final_depth
edge_input_norm = norm_pred_final
#edge prediction for depth
edge_inputs = tf.concat([edges, inputs * 0.00784], axis=3)
edges_encoder = slim.repeat(
edge_inputs,
3,
slim.conv2d,
32, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_edge_refinement')
edges_encoder = slim.repeat(
edges_encoder,
3,
slim.conv2d,
32, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_edge_refinement')
edges_predictor = slim.conv2d(
edges_encoder,
8, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='edge_weight')
edges_all = edges_predictor + tf.tile(edges, [1, 1, 1, 8])
edges_all = tf.clip_by_value(edges_all, 0.0, 1.0)
dlr, drl, dud, ddu, nlr, nrl, nud, ndu = tf.split(
edges_all, num_or_size_splits=8, axis=3)
# 4 iteration depth
final_depth = propagate(edge_input_depth, dlr, drl, dud, ddu,
1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
# 4 iteration norm
norm_pred_final = propagate(edge_input_norm, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
return final_depth, fc8_upsample_norm, norm_pred_final, fc8_upsample
def __init__(self, config):
"""Hyperparameters"""
num_layers = config['num_layers']
hidden_size = config['hidden_size']
max_grad_norm = config['max_grad_norm']
batch_size = config['batch_size']
sl = config['sl']
mixtures = config['mixtures']
crd = config['crd']
learning_rate = config['learning_rate']
MDN = config['MDN']
self.sl = sl
self.crd = crd
self.batch_size = batch_size
# Nodes for the input variables
self.x = tf.placeholder(dtype=tf.float32, shape=[batch_size, crd, sl], name='Input_data')
self.y_ = tf.placeholder(tf.int64, shape=[batch_size], name='Ground_truth')
self.keep_prob = tf.placeholder("float")
with tf.name_scope("LSTM") as scope:
cell = tf.nn.rnn_cell.MultiRNNCell([
lstm_cell(hidden_size, self.keep_prob) for _ in range(num_layers)
])
inputs = tf.unstack(self.x, axis=2)
# outputs, _ = tf.nn.rnn(cell, inputs, dtype=tf.float32)
outputs, _ = tf.contrib.rnn.static_rnn(cell, inputs, dtype=tf.float32)
with tf.name_scope("SoftMax") as scope:
final = outputs[-1]
W_c = tf.Variable(tf.random_normal([hidden_size, 2], stddev=0.01))
b_c = tf.Variable(tf.constant(0.1, shape=[2]))
self.h_c = tf.matmul(final, W_c) + b_c
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.h_c, labels=self.y_)
self.cost = tf.reduce_mean(loss)
loss_summ = tf.summary.scalar("cross entropy_loss", self.cost)
with tf.name_scope("Output_MDN") as scope:
params = 8 # 7+theta
# Two for distribution over hit&miss, params for distribution parameters
output_units = mixtures * params
W_o = tf.Variable(tf.random_normal(
[hidden_size, output_units], stddev=0.01))
b_o = tf.Variable(tf.constant(0.5, shape=[output_units]))
# For comparison with XYZ, only up to last time_step
# --> because for final time_step you cannot make a prediction
output = outputs[:-1]
outputs_tensor = tf.concat(output, axis=0)
# is of size [batch_size*seq_len by output_units]
h_out_tensor = tf.nn.xw_plus_b(outputs_tensor, W_o, b_o)
with tf.name_scope('MDN_over_next_vector') as scope:
# Next two lines are rather ugly, But its the most efficient way to
# reshape the data
h_xyz = tf.reshape(h_out_tensor, (sl - 1, batch_size, output_units))
# transpose to [batch_size, output_units, sl-1]
h_xyz = tf.transpose(h_xyz, [1, 2, 0])
# x_next = tf.slice(x,[0,0,1],[batch_size,3,sl-1]) #in size [batch_size,
# output_units, sl-1]
x_next = tf.subtract(self.x[:, :3, 1:], self.x[:, :3, :sl - 1])
# From here any, many variables have size [batch_size, mixtures, sl-1]
xn1, xn2, xn3 = tf.split(value=x_next, num_or_size_splits=3, axis=1)
self.mu1, self.mu2, self.mu3, self.s1, self.s2, self.s3, self.rho, self.theta = tf.split(value=h_xyz, num_or_size_splits=params, axis=1)
# make the theta mixtures
# softmax all the theta's:
max_theta = tf.reduce_max(self.theta, 1, keep_dims=True)
self.theta = tf.subtract(self.theta, max_theta)
self.theta = tf.exp(self.theta)
normalize_theta = tf.reciprocal(tf.reduce_sum(self.theta, 1, keep_dims=True))
self.theta = tf.multiply(normalize_theta, self.theta)
# Deviances are non-negative and tho between -1 and 1
self.s1 = tf.exp(self.s1)
self.s2 = tf.exp(self.s2)
self.s3 = tf.exp(self.s3)
self.rho = tf.tanh(self.rho)
# probability in x1x2 plane
px1x2 = tf_2d_normal(xn1, xn2, self.mu1, self.mu2,
self.s1, self.s2, self.rho)
px3 = tf_1d_normal(xn3, self.mu3, self.s3)
px1x2x3 = tf.multiply(px1x2, px3)
# Sum along the mixtures in dimension 1
px1x2x3_mixed = tf.reduce_sum(tf.multiply(px1x2x3, self.theta), 1)
print('You are using %.0f mixtures' % mixtures)
# at the beginning, some errors are exactly zero.
loss_seq = -tf.log(tf.maximum(px1x2x3_mixed, 1e-20))
self.cost_seq = tf.reduce_mean(loss_seq)
self.cost_comb = self.cost
if MDN:
# The magic line where both heads come together.
self.cost_comb += self.cost_seq
with tf.name_scope("train") as scope:
tvars = tf.trainable_variables()
# We clip the gradients to prevent explosion
grads = tf.gradients(self.cost_comb, tvars)
grads, _ = tf.clip_by_global_norm(grads, 0.5)
# Some decay on the learning rate
global_step = tf.Variable(0, trainable=False)
lr = tf.train.exponential_decay(
learning_rate, global_step, 14000, 0.95, staircase=True)
optimizer = tf.train.AdamOptimizer(lr)
gradients = zip(grads, tvars)
self.train_step = optimizer.apply_gradients(
gradients, global_step=global_step)
# The following block plots for every trainable variable
# - Histogram of the entries of the Tensor
# - Histogram of the gradient over the Tensor
# - Histogram of the grradient-norm over the Tensor
self.numel = tf.constant([[0]])
for gradient, variable in gradients:
if isinstance(gradient, ops.IndexedSlices):
grad_values = gradient.values
else:
grad_values = gradient
self.numel += tf.reduce_sum(tf.size(variable))
#
# h1 = tf.histogram_summary(variable.name, variable)
# h2 = tf.histogram_summary(variable.name + "/gradients", grad_values)
# h3 = tf.histogram_summary(variable.name + "/gradient_norm", clip_ops.global_norm([grad_values]))
with tf.name_scope("Evaluating_accuracy") as scope:
correct_prediction = tf.equal(tf.argmax(self.h_c, 1), self.y_)
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
accuracy_summary = tf.summary.scalar("accuracy", self.accuracy)
# Define one op to call all summaries
self.merged = tf.summary.merge_all()
def cca_loss(inputs, rcov1=1e-4, rcov2=1e-4, eps=1e-12):
'''
view_t1: nSamples x 2n Bands
view_t2: nSamples x 2n Bands
from https://bitbucket.org/qingming_tang/deep-canonical-correlation-analysis/
'''
eps_eig = eps
N = tf.shape(input=inputs)[0]
d1 = d2 = tf.cast(tf.shape(input=inputs)[1] / 2, tf.int32)
view_t1 = inputs[:, 0:d1]
view_t2 = inputs[:, d1:]
# Remove mean.
m1 = tf.reduce_mean(view_t1, axis=0, keep_dims=True)
view_t1 = tf.subtract(view_t1, m1)
m2 = tf.reduce_mean(view_t2, axis=0, keep_dims=True)
view_t2 = tf.subtract(view_t2, m2)
S11 = tf.matmul(tf.transpose(view_t1), view_t1) / tf.cast(
N - 1, tf.float32) + rcov1 * tf.eye(d1)
S22 = tf.matmul(tf.transpose(view_t2), view_t2) / tf.cast(
N - 1, tf.float32) + rcov2 * tf.eye(d2)
S12 = tf.matmul(tf.transpose(view_t1), view_t2) / tf.cast(
N - 1, tf.float32)
E1, V1 = tf.self_adjoint_eig(S11)
E2, V2 = tf.self_adjoint_eig(S22)
# For numerical stability.
idx1 = tf.where(E1 > eps_eig)[:, 0]
E1 = tf.gather(E1, idx1)
V1 = tf.gather(V1, idx1, axis=1)
idx2 = tf.where(E2 > eps_eig)[:, 0]
E2 = tf.gather(E2, idx2)
V2 = tf.gather(V2, idx2, axis=1)
K11 = tf.matmul(tf.matmul(V1, tf.diag(tf.reciprocal(tf.sqrt(E1)))),
tf.transpose(V1))
K22 = tf.matmul(tf.matmul(V2, tf.diag(tf.reciprocal(tf.sqrt(E2)))),
tf.transpose(V2))
T = tf.matmul(tf.matmul(K11, S12), K22)
# Eigenvalues are sorted in increasing order.
E3, _ = tf.self_adjoint_eig(tf.matmul(T, tf.transpose(T)))
idx3 = tf.where(E3 > eps_eig)[:, 0]
# This is the thresholded rank.
#dim_svd = tf.cond(tf.size(idx3) < outdim, lambda: tf.size(idx3), lambda: outdim)
loss = -tf.reduce_sum(tf.sqrt(E3[-tf.size(idx3):]))
def grad(dy):
S, U, V = tf.linalg.svd(T, full_matrices=True, compute_uv=True)
SS = tf.diag(S)
##
M1 = tf.matmul(K11, U)
M2 = tf.matmul(K22, V)
Delta12 = tf.matmul(M1, M2, transpose_b=True)
Delta11 = -0.5 * tf.matmul(M1, tf.matmul(SS, M1, transpose_b=True))
Delta22 = -0.5 * tf.matmul(M2, tf.matmul(SS, M2, transpose_b=True))
grad1 = 2 * tf.matmul(view_t1, Delta11) + tf.matmul(
view_t2, Delta12, transpose_b=True)
grad1 = grad1 / tf.cast(N - 1, tf.float32)
grad2 = tf.matmul(view_t1, Delta12) + 2 * tf.matmul(view_t2, Delta22)
grad2 = grad2 / tf.cast(N - 1, tf.float32)
return tf.concat((grad1, grad2), axis=-1)
return loss, grad
