def get_loss(self, x1, x2,
tau=0.25, # time step
lbda=0.15, # weight parameter for the data term
theta=0.3, # weight parameter for (u - v)^2
warps=5, # number of warpings per scale
zfactor=0.5, # factor for building the image piramid
max_scales=5, # maximum number of scales for image piramid
max_iterations=5 # maximum number of iterations for optimization
):
u1, u2, rho = self.tvnet_flow(x1, x2,
tau=tau, lbda=lbda, theta=theta, warps=warps,
zfactor=zfactor, max_scales=max_scales,
max_iterations=max_iterations)
# computing loss
u1x, u1y = self.forward_gradient(u1, 'u1')
u2x, u2y = self.forward_gradient(u2, 'u2')
u1_flat = tf.reshape(u1, (tf.shape(x2)[0], 1, x2.shape[1].value * x2.shape[2].value))
u2_flat = tf.reshape(u2, (tf.shape(x2)[0], 1, x2.shape[1].value * x2.shape[2].value))
x2_warp = self.warp_image(x2, u1_flat, u2_flat)
x2_warp = tf.reshape(x2_warp, tf.shape(x2))
loss = lbda * tf.reduce_mean(tf.abs(x2_warp - x1)) + tf.reduce_mean(
tf.abs(u1x) + tf.abs(u1y) + tf.abs(u2x) + tf.abs(u2y))
return loss, u1, u2
def huber_loss(y, y_predicted, m=1.0): """Huber loss.""" t = y - y_predicted # Note that enabling eager execution lets you use Python control flow and # specificy dynamic TensorFlow computations. Contrast this implementation # to the graph-construction one found in `utils`, which uses `tf.cond`. return t ** 2 if tf.abs(t) <= m else m * (2 * tf.abs(t) - m)
def __init__(self,
sess,
dataset_name='facades',
checkpoint_dir=None):
self.sess = sess
self.dataset_name = dataset_name
self.checkpoint_dir = checkpoint_dir
self.real_data = tf.placeholder(tf.float32,
[BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, 3 + 3],
name='input_images')
self.real_A = self.real_data[:, :, :, :3]
self.real_B = self.real_data[:, :, :, 3:6]
self.fake_B = generator(self.real_A, name="generatorA2B")
self.fake_A = generator(self.real_B, name="generatorB2A")
self.fake_B_fake_A = generator(self.fake_B, reuse=True, name="generatorB2A")
self.fake_A_fake_B = generator(self.fake_A, reuse=True, name="generatorA2B")
self.DA_real = discriminator(self.real_A, reuse=False, name="descriminatorA")
self.DB_real = discriminator(self.real_B, reuse=False, name="descriminatorB")
self.DA_fake = discriminator(self.fake_A, reuse=True, name="descriminatorA")
self.DB_fake = discriminator(self.fake_B, reuse=True, name="descriminatorB")
self.g_loss_a2b = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DB_fake, labels=tf.ones_like(self.DB_fake))) + 100 * tf.reduce_mean(
tf.abs(self.real_A - self.fake_B_fake_A)) + 100 * tf.reduce_mean(
tf.abs(self.real_B - self.fake_B))
self.g_loss_b2a = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DA_fake, labels=tf.ones_like(self.DA_fake))) + 100 * tf.reduce_mean(
tf.abs(self.real_B - self.fake_A_fake_B)) + 100 * tf.reduce_mean(
tf.abs(self.real_A - self.fake_A))
self.g_loss = self.g_loss_a2b + self.g_loss_b2a
self.d_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DB_fake, labels=tf.zeros_like(self.DB_fake))) + tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DB_real, labels=tf.ones_like(self.DB_real))) + tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DA_fake, labels=tf.zeros_like(self.DA_fake))) + tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.DA_real, labels=tf.ones_like(self.DA_real)))
self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)
self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)
self.g_loss_a2b_sum = tf.summary.scalar("g_loss_a2b", self.g_loss_a2b)
self.g_loss_b2a_sum = tf.summary.scalar("g_loss_b2a", self.g_loss_b2a)
self.real_A_sum = tf.summary.image("real_A", self.real_A)
self.real_B_sum = tf.summary.image("real_B", self.real_B)
self.fake_A_sum = tf.summary.image("fake_A", self.fake_A)
self.fake_B_sum = tf.summary.image("fake_B", self.fake_B)
self.fake_AB_sum = tf.summary.image("fake_AB", self.fake_A_fake_B)
self.fake_BA_sum = tf.summary.image("fake_BA", self.fake_B_fake_A)
self.d_sum = tf.summary.merge([self.d_loss_sum])
self.g_sum = tf.summary.merge([self.g_loss_sum, self.g_loss_a2b_sum, self.g_loss_b2a_sum,
self.real_A_sum, self.real_B_sum, self.fake_A_sum,
self.fake_B_sum, self.fake_AB_sum, self.fake_BA_sum])
training_vars = tf.trainable_variables()
self.d_vars = [var for var in training_vars if 'd_' in var.name]
self.g_vars = [var for var in training_vars if 'g_' in var.name]
self.saver = tf.train.Saver(max_to_keep=5)
def almost_equal(a, b):
"""
:param a: tensor :param b: tensor
:returns equivalent to numpy: a == b, if a and b were ndarrays
"""
not_almost_equal = tf.abs(tf.sign(tf.round(a - b)))
return tf.abs(not_almost_equal - 1)
def sampled_softmax(tensor, weights):
max_val = tf.reduce_max(tensor * tf.abs(weights), 1, keep_dims=True)
tensor_rescaled = tensor - max_val
tensor_exp = tf.exp(tensor_rescaled)
tensor_sum = tf.reduce_sum(tensor_exp * tf.abs(weights), 1, keep_dims=True)
return (tensor_exp / tensor_sum) * tf.abs(weights) # all ignored elements will have a prob of 0.
def reshape_stft(stfts, num_mel_bins):
magnitude_spectrograms = tf.abs(stfts)
num_spectrogram_bins = magnitude_spectrograms.shape[-1].value
# scale frequency to mel scale and put into bins to reduce dimensionality
lower_edge_hertz, upper_edge_hertz = 30.0, 17000.0
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins, utils.sample_rate, lower_edge_hertz,
upper_edge_hertz)
mel_spectrograms = tf.tensordot(magnitude_spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(
magnitude_spectrograms.shape[:-1].concatenate(linear_to_mel_weight_matrix.shape[-1:]))
# log scale the mel bins to better represent human loudness perception
log_offset = 1e-6
log_mel_spectrograms = tf.log(mel_spectrograms + log_offset)
# compute first order differential and concat. "It indicates a raise or reduction of the energy for each
# frequency bin at a frame relative to its predecessor"
first_order_diff = tf.abs(
tf.subtract(log_mel_spectrograms, tf.manip.roll(log_mel_spectrograms, shift=1, axis=1)))
mel_fod = tf.concat([log_mel_spectrograms, first_order_diff], 1)
return mel_fod
def cycle_consistency_loss(self, G, F, x, y): """ cycle consistency loss (L1 norm) """ forward_loss = tf.reduce_mean(tf.abs(F(G(x))-x)) backward_loss = tf.reduce_mean(tf.abs(G(F(y))-y)) loss = self.lambda1*forward_loss + self.lambda2*backward_loss return loss
def _apply_dense(self, grad, var):
gradients_sum = self.get_slot(var, "gradients_sum")
grad_norm_sum = self.get_slot(var, "grad_norm_sum")
tilde_w = self.get_slot(var, "tilde_w")
L = self.get_slot(var, "L")
reward = self.get_slot(var, "reward")
L_update = tf.maximum(L, tf.abs(grad))
gradients_sum_update = gradients_sum + grad
grad_norm_sum_update = grad_norm_sum + tf.abs(grad)
reward_update = tf.maximum(reward - grad * tilde_w, 0)
new_w = -gradients_sum_update / (
L_update * (tf.maximum(grad_norm_sum_update + L_update, self._alpha * L_update))) * (
reward_update + L_update)
var_update = var - tilde_w + new_w
tilde_w_update = new_w
gradients_sum_update_op = state_ops.assign(gradients_sum, gradients_sum_update)
grad_norm_sum_update_op = state_ops.assign(grad_norm_sum, grad_norm_sum_update)
var_update_op = state_ops.assign(var, var_update)
tilde_w_update_op = state_ops.assign(tilde_w, tilde_w_update)
L_update_op = state_ops.assign(L, L_update)
reward_update_op = state_ops.assign(reward, reward_update)
return control_flow_ops.group(*[gradients_sum_update_op,
var_update_op,
grad_norm_sum_update_op,
tilde_w_update_op,
reward_update_op,
L_update_op])
def huber_loss(x, delta=1.0):
# https://en.wikipedia.org/wiki/Huber_loss
return tf.select(
tf.abs(x) < delta,
tf.square(x) * 0.5,
delta * (tf.abs(x) - 0.5 * delta)
)
def _update_lipschitz(self,v,i):
config = self.config
if len(v.shape) > 1:
k = self.config.weight_constraint_k or 100.0000
wi_hat = v
if len(v.shape) == 4:
#fij = tf.reduce_sum(tf.abs(wi_hat), axis=[0,1])
fij = wi_hat
fij = tf.reduce_sum(tf.abs(fij), axis=[1])
fij = tf.reduce_max(fij, axis=[0])
else:
fij = wi_hat
if self.config.ortho_pnorm == "inf":
wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=0), axis=0)
else:
# conv
wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=1), axis=0)
ratio = (1.0/tf.maximum(1.0, wp/k))
if self.config.weight_bounce:
bounce = tf.minimum(1.0, tf.ceil(wp/k-0.999))
ratio -= tf.maximum(0.0, bounce) * 0.2
if self.config.weight_scaleup:
up = tf.minimum(1.0, tf.ceil(0.02-wp/k))
ratio += tf.maximum(0.0, up) * k/wp * 0.2
wi = ratio*(wi_hat)
#self.gan.metrics['wi'+str(i)]=wp
#self.gan.metrics['wk'+str(i)]=ratio
#self.gan.metrics['bouce'+str(i)]=bounce
return tf.assign(v, wi)
return None
def _encode(self, boxes, anchors):
"""Encodes a box collection with respect to an anchor collection.
Args:
boxes: BoxList holding N boxes to be encoded.
anchors: BoxList of anchors.
Returns:
a tensor representing N anchor-encoded boxes of the format
[ty, tx, tl].
"""
# Convert anchors to the center coordinate representation.
ycenter_a, xcenter_a, ha, wa = anchors.get_center_coordinates_and_sizes()
la = tf.sqrt(ha * wa)
ycenter, xcenter, h, w = boxes.get_center_coordinates_and_sizes()
l = tf.sqrt(h * w)
# Avoid NaN in division and log below.
la += EPSILON
l += EPSILON
top = tf.abs(ycenter_a - ycenter + 0.5*h)
bown = tf.abs(ycenter_a - ycenter - 0.5*h)
left = tf.abs(xcenter_a - xcenter + 0.5*w)
right = tf.abs(xcenter_a - xcenter - 0.5*w)
# Scales location targets for joint training.
if self._scale_factors:
top *= self._scale_factors[0]
bown *= self._scale_factors[0]
left *= self._scale_factors[1]
right *= self._scale_factors[1]
return tf.transpose(tf.stack([top, bown, left, right]))
def _build_loss(self):
with tf.variable_scope("loss"):
# Compute y_j = r_j * discount*best_qvalue
self.tf_discount = tf.constant(self.discount)
self.qtarget = tf.add(self.pl_rewards, tf.mul(1.0-self.pl_terminals, tf.mul(self.tf_discount, self.pl_qtargets)))
# Select Q-values for given actions
self.actions_one_hot = tf.one_hot(self.pl_actions, self.num_actions, 1.0, 0.0)
self.qvalue_pred = tf.reduce_sum(tf.mul(self.qvalues, self.actions_one_hot), reduction_indices=1)
# Difference between target and predicted Q-network output
self.delta = tf.sub(self.qtarget, self.qvalue_pred)
if self.clip_delta > 0:
# Perform clipping of the error term, default clipping is to (-1, +1) range
self.quadratic_part = tf.minimum(tf.abs(self.delta), tf.constant(self.clip_delta))
self.linear_part = tf.sub(tf.abs(self.delta), self.quadratic_part)
self.delta_square = tf.mul(tf.constant(0.5), tf.square(self.quadratic_part)) + (self.clip_delta*self.linear_part)
#self.delta_clipped = tf.clip_by_value(self.delta, -1.0*self.clip_delta, self.clip_delta)
#self.delta_square = tf.square(self.delta_clipped)
else:
# No error clipping
self.delta_square = tf.square(self.delta)
# Actual loss
if self.batch_accumulator == "sum":
self.loss = tf.reduce_sum(self.delta_square)
else:
self.loss = tf.reduce_mean(self.delta_square)
# Running average of the loss for TensorBoard
self.loss_moving_avg = tf.train.ExponentialMovingAverage(decay=0.999)
self.loss_moving_avg_op = self.loss_moving_avg.apply([self.loss])
def check_convergence(self, new_T0, new_transition, new_emission):
delta_T0 = tf.reduce_max(tf.abs(self.T0 - new_T0)) < self.epsilon
delta_T = tf.reduce_max(tf.abs(self.T - new_transition)) < self.epsilon
delta_E = tf.reduce_max(tf.abs(self.E - new_emission)) < self.epsilon
return tf.logical_and(tf.logical_and(delta_T0, delta_T), delta_E)
def huber_loss(x, delta=1.0):
"""Reference: https://en.wikipedia.org/wiki/Huber_loss"""
return tf.where(
tf.abs(x) < delta,
tf.square(x) * 0.5,
delta * (tf.abs(x) - 0.5 * delta)
)
def cross_entropy(u, label_u, alpha=0.5, normed=False):
label_ip = tf.cast(
tf.matmul(label_u, tf.transpose(label_u)), tf.float32)
s = tf.clip_by_value(label_ip, 0.0, 1.0)
# compute balance param
# s_t \in {-1, 1}
s_t = tf.multiply(tf.add(s, tf.constant(-0.5)), tf.constant(2.0))
sum_1 = tf.reduce_sum(s)
sum_all = tf.reduce_sum(tf.abs(s_t))
balance_param = tf.add(tf.abs(tf.add(s, tf.constant(-1.0))),
tf.multiply(tf.div(sum_all, sum_1), s))
if normed:
# ip = tf.clip_by_value(tf.matmul(u, tf.transpose(u)), -1.5e1, 1.5e1)
ip_1 = tf.matmul(u, tf.transpose(u))
def reduce_shaper(t):
return tf.reshape(tf.reduce_sum(t, 1), [tf.shape(t)[0], 1])
mod_1 = tf.sqrt(tf.matmul(reduce_shaper(tf.square(u)),
reduce_shaper(tf.square(u)), transpose_b=True))
ip = tf.div(ip_1, mod_1)
else:
ip = tf.clip_by_value(tf.matmul(u, tf.transpose(u)), -1.5e1, 1.5e1)
ones = tf.ones([tf.shape(u)[0], tf.shape(u)[0]])
return tf.reduce_mean(tf.multiply(tf.log(ones + tf.exp(alpha * ip)) - s * alpha * ip, balance_param))
def get_train(train_ph_dict,var_dict,var_ph_dict):
mid0 = tf.one_hot(train_ph_dict['choice_0'], 9, axis=-1, dtype=tf.float32)
mid0 = mid0 * get_q(train_ph_dict['state_0'],var_dict)
mid0 = tf.reduce_sum(mid0, reduction_indices=[1])
mid1 = get_q(train_ph_dict['state_1'],var_ph_dict)
mid1 = tf.reduce_max(mid1, reduction_indices=[1])
mid1 = mid1 * train_ph_dict['cont']
mid1 = mid1 * tf.constant(TRAIN_BETA)
l2r = tf.constant(0.0)
cell_count = tf.constant(0.0)
for v in var_dict.values():
l2r = l2r + get_l2(v)
cell_count = cell_count + tf.to_float(tf.size(v))
l2r = l2r / cell_count
l2r = l2r / tf.constant(ELEMENT_L2_FACTOR*ELEMENT_L2_FACTOR)
l2r = l2r * tf.constant(L2_WEIGHT)
mid = mid0-mid1-train_ph_dict['reward_1']
# mid = mid * mid
mid = tf.abs(mid)
mid = tf.reduce_mean(mid)
score_diff = mid
mid = mid + l2r
mid = mid + ( tf.abs( tf.reduce_mean(var_dict['b5']) ) * tf.constant(L2_WEIGHT) )
loss = mid
mid = tf.train.GradientDescentOptimizer(0.5).minimize(mid,var_list=var_dict.values())
train = mid
return train, loss, score_diff
def __init__(self, session, np_matrix, rank,
learning_rate=0.1):
matrix = tf.constant(np_matrix, dtype=tf.float32)
scale = 2 * np.sqrt(np_matrix.mean() / rank)
initializer = tf.random_uniform_initializer(maxval=scale)
with tf.device('/job:ps/task:0'):
self.matrix_W = tf.get_variable(
"W", (np_matrix.shape[0], rank), initializer=initializer
)
with tf.device("/job:ps/task:1"):
self.matrix_H = tf.get_variable(
"H", (rank, np_matrix.shape[1]), initializer=initializer
)
matrix_WH = tf.matmul(self.matrix_W, self.matrix_H)
f_norm = tf.reduce_sum(tf.pow(matrix - matrix_WH, 2))
nn_w = tf.reduce_sum(tf.abs(self.matrix_W) - self.matrix_W)
nn_h = tf.reduce_sum(tf.abs(self.matrix_H) - self.matrix_H)
constraint = INFINITY * (nn_w + nn_h)
self.loss = f_norm + constraint
self.constraint = constraint
self.session = session
self.optimizer = tf.train.GradientDescentOptimizer(
learning_rate
).minimize(self.loss)
def attention_mechanism_parallel(self,c_full,m,q,i):
""" parallel implemtation of gate function given a list of candidate sentence, a query, and previous memory.
Input:
c_full: candidate fact. shape:[batch_size,story_length,hidden_size]
m: previous memory. shape:[batch_size,hidden_size]
q: question. shape:[batch_size,hidden_size]
Output: a scalar score (in batch). shape:[batch_size,story_length]
"""
q=tf.expand_dims(q,axis=1) #[batch_size,1,hidden_size]
m=tf.expand_dims(m,axis=1) #[batch_size,1,hidden_size]
# 1.define a large feature vector that captures a variety of similarities between input,memory and question vector: z(c,m,q)
c_q_elementwise=tf.multiply(c_full,q) #[batch_size,story_length,hidden_size]
c_m_elementwise=tf.multiply(c_full,m) #[batch_size,story_length,hidden_size]
c_q_minus=tf.abs(tf.subtract(c_full,q)) #[batch_size,story_length,hidden_size]
c_m_minus=tf.abs(tf.subtract(c_full,m)) #[batch_size,story_length,hidden_size]
# c_transpose Wq
c_w_q=self.x1Wx2_parallel(c_full,q,"c_w_q"+str(i)) #[batch_size,story_length,hidden_size]
c_w_m=self.x1Wx2_parallel(c_full,m,"c_w_m"+str(i)) #[batch_size,story_length,hidden_size]
# c_transposeWm
q_tile=tf.tile(q,[1,self.story_length,1]) #[batch_size,story_length,hidden_size]
m_tile=tf.tile(m,[1,self.story_length,1]) #[batch_size,story_length,hidden_size]
z=tf.concat([c_full,m_tile,q_tile,c_q_elementwise,c_m_elementwise,c_q_minus,c_m_minus,c_w_q,c_w_m],2) #[batch_size,story_length,hidden_size*9]
# 2. two layer feed foward
g=tf.layers.dense(z,self.hidden_size*3,activation=tf.nn.tanh) #[batch_size,story_length,hidden_size*3]
g=tf.layers.dense(g,1,activation=tf.nn.sigmoid) #[batch_size,story_length,1]
g=tf.squeeze(g,axis=2) #[batch_size,story_length]
return g
def add_dyprune(weights):
crate = config.crate[weights.name[:-2]] #hyperpara C rate
prune_mask = tf.Variable(tf.ones_like(weights),name=weights.name[:-2]+'mask', trainable=False)
#calculate mask
mean = tf.divide(tf.reduce_sum(tf.multiply(tf.abs(weights),prune_mask)),tf.reduce_sum(prune_mask))
var = tf.multiply(weights,prune_mask)
var = tf.square(var)
mean_q = tf.square(mean)*tf.reduce_sum(prune_mask)
var = tf.reduce_sum(var) - mean_q
var = tf.divide(var,tf.reduce_sum(prune_mask))
var = tf.sqrt(var)
t1_lower = (mean+var*crate)*0.25 #hyperpara a
t1_upper = (mean+var*crate)*0.45 #hyperpara b
indicator_lower1 = tf.greater_equal(tf.abs(weights), tf.ones_like(weights) * t1_lower)
indicator_upper1 = tf.greater_equal(tf.abs(weights), tf.ones_like(weights) * t1_upper)
indicator_matrix1 = tf.greater_equal(prune_mask, tf.zeros_like(weights))
indicator_matrix1 = tf.logical_and(indicator_matrix1,indicator_lower1)
indicator_matrix1 = tf.logical_or(indicator_matrix1,indicator_upper1)
indicator_matrix1 = tf.to_float(indicator_matrix1)
update = prune_mask.assign(indicator_matrix1)
prune_fc = tf.multiply(weights, prune_mask)
return prune_fc
def __init__(self, inpt, n_in, n_hidden, n_out):
"""
inpt: tf.Tensor, shape [n_examples, n_in]
n_in: int, the dimensionality of input
n_hidden: int, number of hidden units
n_out: int, number of output units
"""
# hidden layer
self.hiddenLayer = HiddenLayer(inpt, n_in=n_in, n_out=n_hidden)
# output layer (logistic layer)
self.outputLayer = LogisticRegression(self.hiddenLayer.output, n_in=n_hidden,
n_out=n_out)
# L1 norm
self.L1 = tf.reduce_sum(tf.abs(self.hiddenLayer.W)) + \
tf.reduce_sum(tf.abs(self.outputLayer.W))
# L2 norm
self.L2 = tf.reduce_sum(tf.square(self.hiddenLayer.W)) + \
tf.reduce_sum(tf.square(self.outputLayer.W))
# cross_entropy cost function
self.cost = self.outputLayer.cost
# accuracy function
self.accuracy = self.outputLayer.accuarcy
# params
self.params = self.hiddenLayer.params + self.outputLayer.params
# keep track of input
self.input = inpt
def NTanh(x,
use_noise,
alpha=1.05,
c=0.5, half_normal=False):
"""
Noisy Hard Tanh Units: NAN without learning p
----------------------------------------------------
Arguments:
x: tensorflow tensor variable, input of the function.
use_noise: bool, whether to add noise or not to the activations, this is in particular
useful for the test time, in order to disable the noise injection.
c: float, standard deviation of the noise
alpha: the leaking rate from the linearized function to the nonlinear one.
"""
threshold = 1.0
signs = tf.sign(x)
delta = tf.abs(x) - threshold
scale = c * (tf.sigmoid(delta**2) - 0.5)**2
if alpha > 1.0 and half_normal:
scale *= -1.0
zeros=tf.zeros(tf.shape(x), dtype=tf.float32, name=None)
def noise_func() :return tf.abs(tf.random_normal(tf.shape(x), mean=0.0, stddev=1.0, dtype=tf.float32))
def zero_func (): return zeros+ 0.797 if half_normal else zeros
noise=tf.cond(use_noise,noise_func,zero_func)
eps = scale * noise + alpha * delta
z = x - signs * eps
test=tf.cast(tf.greater_equal(tf.abs(x) , threshold),tf.float32)
res = test * z + (1. - test) * HardTanh(x)
return res
def __loss__(self):
"""
Calculate loss
:return:
"""
# Context loss L2
predict_image = tf.abs(tf.complex(real=self.predict_g2['real'], imag=self.predict_g2['imag']))
label_image = tf.abs(tf.complex(real=self.labels['real'], imag=self.labels['imag']))
self.context_loss = tf.reduce_mean(tf.square(tf.contrib.layers.flatten(predict_image - label_image)))
# self.context_loss = tf.reduce_mean(tf.square(real_diff) + tf.square(imag_diff), name='Context_loss_mean')
print("You are using L2 loss")
tf.summary.scalar('g_loss_context_only', self.context_loss, collections='G2')
self.g_loss = self.FLAGS.gen_loss_context * self.context_loss
# self.g_loss = self.FLAGS.gen_loss_adversarial * g_loss + self.FLAGS.gen_loss_context * context_loss
tf.summary.scalar('g_loss_plus_context', self.g_loss, collections='G2')
# if len(self.regularization_values) > 0:
# reg_loss_g = self.reg_w * tf.reduce_sum(self.regularization_values)
self.reg_loss_g = self.get_weights_regularization(dump=self.FLAGS.dump_debug, collection='G2')
self.g_loss_no_reg = self.g_loss
self.g_loss += self.reg_loss_g
if self.FLAGS.dump_debug:
tf.summary.scalar('g_loss_plus_context_plus_reg', self.g_loss, collections='G2')
tf.summary.scalar('g_loss_reg_only', self.reg_loss_g, collections='D')
def __init__(self, shape, lambda1 = 0.1, lambda2 = 0.1, mu = 0.1):
"""Initialize the ChanVese segmenter
Arguments:
shape (required) -- size of the image to segment
lambda1 (default : 0.1) -- The cost of labeling pixels type 1 (check the Class docstring). This argument (as well as lambda2) can be used if the segmentation should be biased in one direction or the other. It's not deterministic what bits of the image get labeled with either lambda though -- this (as well as lambda2) will likely be a bit of a guess and check parameter.
lambda2 (default : 0.1) -- The cost of labeling pixels type 2 (check the Class docstring)
mu (default : 0.1) -- This is the cost of having a boundary. A higher value will mean less boundaries
"""
xs = range(3)
ys = range(3)
Xs, Ys = numpy.meshgrid(xs, ys)
Rs = numpy.sqrt((Xs - 1.0)**2 + (Ys - 1.0)**2)
kernelBlurCpu = numpy.exp(-Rs / (2.0 * 0.75**2)).astype('float32')
kernelBlurCpu /= numpy.linalg.norm(kernelBlurCpu.flatten())
self.kernel = tf.constant(kernelBlurCpu.reshape([3, 3, 1, 1]))
self.I = tf.Variable(tf.truncated_normal(shape = [1, shape[0], shape[1], 1], mean = 0.0, stddev = 0.1))
self.u1 = tf.Variable(1.0)
self.u2 = tf.Variable(-1.0)
self.G = tf.placeholder(tf.float32, shape = shape)
self.Gv = tf.Variable(numpy.zeros([1, shape[0], shape[1], 1]).astype('float32'))
self.initialize = self.Gv.assign(tf.reshape(self.G, shape = [1, shape[0], shape[1], 1]))
self.initialize2 = self.I.assign(tf.reshape(self.G, shape = [1, shape[0], shape[1], 1]))
self.blur = tf.nn.conv2d(self.I, self.kernel, strides = [1, 1, 1, 1], padding = 'SAME')
self.Gv = tf.Variable(numpy.zeros([1, shape[0], shape[1], 1]).astype('float32'))
self.u1m = tf.abs(self.blur - self.u1)
self.u2m = tf.abs(self.blur - self.u2)
ones = numpy.ones((1, shape[0], shape[1], 1)).astype('float32')
zeros = numpy.zeros((1, shape[0], shape[1], 1)).astype('float32')
self.lambda1 = lambda1
self.lambda2 = lambda2
self.mu = mu
eta = 0.1
self.conv = eta / (numpy.pi * (eta**2 + self.blur**2))
self.u1t = self.lambda1 * tf.reduce_sum(tf.select(self.u2m > self.u1m, (self.Gv - self.u1)**2, zeros))
self.u2t = self.lambda2 * tf.reduce_sum(tf.select(self.u2m <= self.u1m, (self.Gv - self.u2)**2, zeros))
self.edgeLoss = self.mu * tf.reduce_sum(tf.abs(self.conv))
self.loss = self.u1t + self.u2t + self.edgeLoss
self.shape = shape
self.train_step = tf.train.AdamOptimizer(1.0e-1).minimize(self.loss, var_list = [self.I, self.u1, self.u2])
def crop_or_pad(image, curr_height, curr_width, new, height=True, crop=True):
"""Crops or pads an image.
Args:
image: 3-D float32 `Tensor` image.
curr_height: Int, current height.
curr_width: Int, current width.
new: Int, new width or height.
height: Boolean, cropping or padding for height.
crop: Boolean, True if we're cropping, False if we're padding.
Returns:
image: 3-D float32 `Tensor` image.
"""
# Crop the image to fit the new shape.
abs_diff = tf.abs(new-curr_height)//2 if height else tf.abs(new-curr_width)//2
offset_x = 0 if height else abs_diff
offset_y = abs_diff if height else 0
# We process height first, so always pad/crop to new height.
target_height = new
# We process height first, so pad/crop to new width only if not doing height.
target_width = curr_width if height else new
if crop:
image = tf.image.crop_to_bounding_box(
image, offset_y, offset_x, target_height, target_width)
else:
image = tf.image.pad_to_bounding_box(
image, offset_y, offset_x, target_height, target_width)
return image
def build(self, X, y):
"""
Builds the neural network in tensorflow
"""
## First, create a placeholder for the targets
self.targets = tf.placeholder(tf.float32, shape=[None, self.n_categories])
## First, create the input layer
self.input_layer = Layer(n_neurons=self.n_features)
self.input_layer.build()
## Then create all the hidden layers
current_input_layer = self.input_layer
for layer in self.hidden_layers:
layer.build(current_input_layer)
current_input_layer = layer
## Create the output layer
self.output_layer = Layer(n_neurons=self.n_categories, activation=self.output_activation)
self.output_layer.build(current_input_layer)
## Define the cost function
self.cost = None
if self.cost_function == 'log-likelihood':
self.cost = tf.reduce_mean(-tf.log(1e-37 + tf.reduce_sum(self.targets * self.output_layer.output, reduction_indices=[1])))
else: ## cross-entropy
self.cost = tf.reduce_mean(-tf.reduce_sum(self.targets * tf.log(1e-37 + self.output_layer.output), reduction_indices=[1]))
## Define the regularization parameters and function
self.reg_lambda_param = tf.placeholder(tf.float32)
self.batch_size = tf.placeholder(tf.float32)
if self.regularization == 'l1':
self.reg_term = tf.reduce_sum(tf.abs(self.output_layer.weights))
for layer in self.hidden_layers:
self.reg_term += tf.reduce_sum(tf.abs(layer.weights))
elif self.regularization == 'l2':
self.reg_term = tf.reduce_sum(self.output_layer.weights * self.output_layer.weights)
for layer in self.hidden_layers:
self.reg_term += tf.reduce_sum(layer.weights * layer.weights)
else:
self.reg_term = None
## Add the regularization term to the cost function
if self.reg_term is None:
self.reg_cost = self.cost
else:
self.reg_cost = self.cost + (self.reg_lambda_param/(2*self.batch_size))*self.reg_term
## Define the train step
self.train_step = getattr(tf.train, '{0}Optimizer'.format(self.learning_algorithm))(self.learning_rate).minimize(self.reg_cost)
## Initialize everything
self.session.run(tf.initialize_all_variables())
def visualize_data_transformations():
records = glob.glob(os.path.join(utils.working_dir, 'train_fragment_*.tfrecords'))
dataset = tf.data.TFRecordDataset(records)
dataset = dataset.map(parse_tfrecord_raw)
dataset = dataset.repeat()
dataset = dataset.shuffle(buffer_size=10)
dataset = dataset.prefetch(2)
it = dataset.make_one_shot_iterator()
data_x = tf.placeholder(tf.float32, shape=(utils.sample_rate * utils.audio_clip_len,))
data_y = tf.placeholder(tf.float32, shape=(utils.timesteps,))
stfts = tf.contrib.signal.stft(data_x, frame_length=utils.frame_length, frame_step=utils.frame_step,
fft_length=4096)
power_stfts = tf.real(stfts * tf.conj(stfts))
magnitude_spectrograms = tf.abs(stfts)
power_magnitude_spectrograms = tf.abs(power_stfts)
num_spectrogram_bins = magnitude_spectrograms.shape[-1].value
# scale frequency to mel scale and put into bins to reduce dimensionality
lower_edge_hertz, upper_edge_hertz = 30.0, 17000.0
num_mel_bins = utils.mel_bins_base * 4
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins, utils.sample_rate, lower_edge_hertz,
upper_edge_hertz)
mel_spectrograms = tf.tensordot(magnitude_spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(
magnitude_spectrograms.shape[:-1].concatenate(linear_to_mel_weight_matrix.shape[-1:]))
# log scale the mel bins to better represent human loudness perception
log_offset = 1e-6
log_mel_spectrograms = tf.log(mel_spectrograms + log_offset)
# compute first order differential and concat. "It indicates a raise or reduction of the energy for each
# frequency bin at a frame relative to its predecessor"
first_order_diff = tf.abs(
tf.subtract(log_mel_spectrograms, tf.manip.roll(log_mel_spectrograms, shift=1, axis=1)))
mel_fod = tf.concat([log_mel_spectrograms, first_order_diff], 1)
with tf.Session() as sess:
while True:
try:
raw_x, raw_y = sess.run(it.get_next())
np_stfts = sess.run(power_stfts, feed_dict={data_x: raw_x})
np_magnitude_spectrograms = sess.run(power_magnitude_spectrograms, feed_dict={data_x: raw_x})
np_mel_spectrograms = sess.run(mel_spectrograms, feed_dict={data_x: raw_x})
np_log_mel_spectrograms = sess.run(log_mel_spectrograms, feed_dict={data_x: raw_x})
np_mel_fod = sess.run(mel_fod, feed_dict={data_x: raw_x})
utils.plot_signal_transforms(raw_x,
np_stfts,
np_magnitude_spectrograms,
np_mel_spectrograms,
np_log_mel_spectrograms,
np_mel_fod)
print('wank')
except tf.errors.OutOfRangeError:
break
def build_model(self):
self.is_training = tf.placeholder(tf.bool, name='is_training')
self.images = tf.placeholder(
tf.float32, [None] + self.image_shape, name='real_images')
self.lowres_images = tf.reduce_mean(tf.reshape(self.images,
[self.batch_size, self.lowres_size, self.lowres,
self.lowres_size, self.lowres, self.c_dim]), [2, 4])
self.z = tf.placeholder(tf.float32, [None, self.z_dim], name='z')
self.z_sum = tf.summary.histogram("z", self.z)
self.G = self.generator(self.z)
self.lowres_G = tf.reduce_mean(tf.reshape(self.G,
[self.batch_size, self.lowres_size, self.lowres,
self.lowres_size, self.lowres, self.c_dim]), [2, 4])
self.D, self.D_logits = self.discriminator(self.images)
self.D_, self.D_logits_ = self.discriminator(self.G, reuse=True)
self.d_sum = tf.summary.histogram("d", self.D)
self.d__sum = tf.summary.histogram("d_", self.D_)
self.G_sum = tf.summary.image("G", self.G)
self.d_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits,
labels=tf.ones_like(self.D)))
self.d_loss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_,
labels=tf.zeros_like(self.D_)))
self.g_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_,
labels=tf.ones_like(self.D_)))
self.d_loss_real_sum = tf.summary.scalar("d_loss_real", self.d_loss_real)
self.d_loss_fake_sum = tf.summary.scalar("d_loss_fake", self.d_loss_fake)
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)
self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.saver = tf.train.Saver(max_to_keep=1)
# Completion.
self.mask = tf.placeholder(tf.float32, self.image_shape, name='mask')
self.lowres_mask = tf.placeholder(tf.float32, self.lowres_shape, name='lowres_mask')
self.contextual_loss = tf.reduce_sum(
tf.contrib.layers.flatten(
tf.abs(tf.multiply(self.mask, self.G) - tf.multiply(self.mask, self.images))), 1)
self.contextual_loss += tf.reduce_sum(
tf.contrib.layers.flatten(
tf.abs(tf.multiply(self.lowres_mask, self.lowres_G) - tf.multiply(self.lowres_mask, self.lowres_images))), 1)
self.perceptual_loss = self.g_loss
self.complete_loss = self.contextual_loss + self.lam*self.perceptual_loss
self.grad_complete_loss = tf.gradients(self.complete_loss, self.z)
def pseudocount_near_zero(tensor):
return tensor + (NEAR_ZERO_THRESHOLD*(lt_mask(tf.abs(tensor),
0.5*NEAR_ZERO_THRESHOLD)*
gte_mask(tensor,0)) -
NEAR_ZERO_THRESHOLD*(lt_mask(tf.abs(tensor),
0.5*NEAR_ZERO_THRESHOLD)*
lt_mask(tensor,0)))
def __D__(self, input_d):
"""
Define the discriminator
"""
# Input d holds real&imaginary values. The discriminative decision based on reconstructed image
input_to_discriminator = input_d
org = input_to_discriminator[0]
fake = input_to_discriminator[1]
rec_org = tf.abs(tf.expand_dims(input=tf.complex(real=org[:, 0, :, :], imag=org[:, 1, :, :]), dim=1))
rec_fake = tf.abs(tf.expand_dims(input=tf.complex(real=fake[:, 0, :, :], imag=fake[:, 1, :, :]), dim=1))
tf.summary.image('D_x_input_reconstructed' + 'Original', tf.transpose(rec_org, (0,2,3,1)), collections='D', max_outputs=4)
tf.summary.image('D_x_input_reconstructed' + 'Fake', tf.transpose(rec_fake, (0,2,3,1)), collections='G2', max_outputs=4)
input_to_discriminator = tf.concat(input_to_discriminator, axis=0)
# Model convolutions
out_dim = 8 # 128x128
self.conv_1_d = ops.conv2d(input_to_discriminator, output_dim=out_dim, k_h=3, k_w=3, d_h=1, d_w=1, name="D_conv_1")
self.pool_1_d = tf.layers.max_pooling2d(self.conv_1_d, pool_size=[2, 2], strides=2, padding='same',
data_format='channels_first',name="D_pool_1")
self.conv_1_bn_d = ops.batch_norm(self.pool_1_d, self.train_phase, decay=0.98, name="D_bn1")
# self.relu_1_d = tf.nn.relu(self.conv_1_bn_d)
self.relu_1_d = ops.lrelu(self.conv_1_bn_d, name="D_relu1")
out_dim = 16 # 64x64
self.conv_2_d = ops.conv2d(self.relu_1_d, output_dim=out_dim, k_h=3, k_w=3, d_h=1, d_w=1,
name="D_conv_2")
self.pool_2_d = tf.layers.max_pooling2d(self.conv_2_d, pool_size=[2, 2], strides=2, padding='same',
data_format='channels_first',name="D_pool_2")
self.conv_2_bn_d = ops.batch_norm(self.pool_2_d, self.train_phase, decay=0.98, name="D_bn2")
# self.relu_2_d = tf.nn.relu(self.conv_2_bn_d)
self.relu_2_d = ops.lrelu(self.conv_2_bn_d, name="D_relu2")
# out_dim = 32 # 32x32
out_dim = 8 # 32x32
self.conv_3_d = ops.conv2d(self.relu_2_d, output_dim=out_dim, k_h=3, k_w=3, d_h=1, d_w=1,
name="D_conv_3")
self.pool_3_d = tf.layers.max_pooling2d(self.conv_3_d, pool_size=[2, 2], strides=2, padding='same',
data_format='channels_first',name="D_pool_3")
self.conv_3_bn_d = ops.batch_norm(self.pool_3_d, self.train_phase, decay=0.98, name="D_bn3")
# self.relu_3_d = tf.nn.relu(self.conv_3_bn_d)
self.relu_3_d = ops.lrelu(self.conv_3_bn_d, name="D_relu3")
# out_dim = 16 # 16x16
# self.conv_4_d = ops.conv2d(self.relu_3_d, output_dim=out_dim, k_h=3, k_w=3, d_h=1, d_w=1,
# name="D_conv_4")
#self.pool_4_d = tf.layers.max_pooling2d(self.conv_4_d, pool_size=[2, 2], strides=2, padding='same',
# data_format='channels_first',name="D_pool_4")
# self.conv_4_bn_d = ops.batch_norm(self.pool_4_d, self.train_phase, decay=0.98, name="D_bn4")
# # self.relu_4_d = tf.nn.relu(self.conv_4_bn_d)
# self.relu_4_d = ops.lrelu(self.conv_4_bn_d)
out_dim = 1
self.affine_1_d = ops.linear(tf.contrib.layers.flatten(self.relu_3_d), output_size=out_dim, scope="D_affine_1")
predict_d = self.affine_1_d
# Dump prediction out
return tf.nn.sigmoid(predict_d), predict_d
def E_reg(I, alpha):
alpha_ = tf.expand_dims(alpha, 0)
ax = tf.nn.conv2d(alpha_, sobel_x_filter, strides=[1, 1, 1, 1], padding="SAME")
ay = tf.nn.conv2d(alpha_, sobel_y_filter, strides=[1, 1, 1, 1], padding="SAME")
Ix2 = tf.square(tf.nn.conv2d(I, sobel_x_filter, strides=[1, 1, 1, 1], padding="SAME"))
Iy2 = tf.square(tf.nn.conv2d(I, sobel_y_filter, strides=[1, 1, 1, 1], padding="SAME"))
est_error = tf.multiply(tf.abs(ax), Ix2) + tf.multiply(tf.abs(ay), Iy2)
est_error = tf.reduce_mean(phi_func(est_error))
return est_error
def abs_smooth(self, x):
absx = tf.abs(x)
minx = tf.minimum(absx, 1)
r = 0.5 * ((absx - 1) * minx + absx)
return r
def build_network(self, sess, is_training=True):
# select initializers
if cfg.TRAIN.TRUNCATED:
initializer = tf.truncated_normal_initializer(mean=0.0,
stddev=0.01)
initializer_bbox = tf.truncated_normal_initializer(mean=0.0,
stddev=0.001)
else:
#initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)
initializer = tf.contrib.layers.xavier_initializer()
initializer_bbox = tf.random_normal_initializer(mean=0.0,
stddev=0.001)
bottleneck = resnet_v1.bottleneck
# choose different blocks for different number of layers
if self._num_layers == 50:
blocks = [
resnet_utils.Block('block1', bottleneck,
[(256, 64, 1)] * 2 + [(256, 64, 2)]),
resnet_utils.Block('block2', bottleneck,
[(512, 128, 1)] * 3 + [(512, 128, 2)]),
# Use stride-1 for the last conv4 layer
resnet_utils.Block('block3', bottleneck,
[(1024, 256, 1)] * 5 + [(1024, 256, 1)]),
resnet_utils.Block('block4', bottleneck, [(2048, 512, 1)] * 3)
]
elif self._num_layers == 101:
blocks = [
resnet_utils.Block('block1', bottleneck,
[(256, 64, 1)] * 2 + [(256, 64, 2)]),
resnet_utils.Block('block2', bottleneck,
[(512, 128, 1)] * 3 + [(512, 128, 2)]),
# Use stride-1 for the last conv4 layer
resnet_utils.Block('block3', bottleneck,
[(1024, 256, 1)] * 22 + [(1024, 256, 1)]),
resnet_utils.Block('block4', bottleneck, [(2048, 512, 1)] * 3)
]
elif self._num_layers == 152:
blocks = [
resnet_utils.Block('block1', bottleneck,
[(256, 64, 1)] * 2 + [(256, 64, 2)]),
resnet_utils.Block('block2', bottleneck,
[(512, 128, 1)] * 7 + [(512, 128, 2)]),
# Use stride-1 for the last conv4 layer
resnet_utils.Block('block3', bottleneck,
[(1024, 256, 1)] * 35 + [(1024, 256, 1)]),
resnet_utils.Block('block4', bottleneck, [(2048, 512, 1)] * 3)
]
else:
# other numbers are not supported
raise NotImplementedError
assert (0 <= cfg.RESNET.FIXED_BLOCKS < 4)
if cfg.RESNET.FIXED_BLOCKS == 3:
with slim.arg_scope(resnet_arg_scope(is_training=False)):
net = self.build_base()
net_conv4, _ = resnet_v1.resnet_v1(
net,
blocks[0:cfg.RESNET.FIXED_BLOCKS],
global_pool=False,
include_root_block=False,
scope=self._resnet_scope)
elif cfg.RESNET.FIXED_BLOCKS > 0:
with slim.arg_scope(resnet_arg_scope(is_training=False)):
net = self.build_base()
net, _ = resnet_v1.resnet_v1(net,
blocks[0:cfg.RESNET.FIXED_BLOCKS],
global_pool=False,
include_root_block=False,
scope=self._resnet_scope)
with slim.arg_scope(resnet_arg_scope(is_training=is_training)):
net_conv4, _ = resnet_v1.resnet_v1(
net,
blocks[cfg.RESNET.FIXED_BLOCKS:-1],
global_pool=False,
include_root_block=False,
scope=self._resnet_scope)
else: # cfg.RESNET.FIXED_BLOCKS == 0
with slim.arg_scope(resnet_arg_scope(is_training=is_training)):
net = self.build_base()
net_conv4, _ = resnet_v1.resnet_v1(net,
blocks[0:-1],
global_pool=False,
include_root_block=False,
scope=self._resnet_scope)
self._act_summaries.append(net_conv4)
self._layers['head'] = net_conv4
c = np.zeros((3, 5, 5))
c[0] = [[-1, 2, -2, 2, -1], [2, -6, 8, -6, 2], [-2, 8, -12, 8, -2],
[2, -6, 8, -6, 2], [-1, 2, -2, 2, -1]]
c[0] = c[0] / 12
c[1][1][1] = -1
c[1][1][2] = 2
c[1][1][3] = -1
c[1][2][1] = 2
c[1][2][2] = -4
c[1][2][3] = 2
c[1][3][1] = -1
c[1][3][2] = 2
c[1][3][3] = -1
c[1] = c[1] / 4
c[2][2][1] = 1
c[2][2][2] = -2
c[2][2][3] = 1
c[2] = c[2] / 2
Wcnn = np.zeros((5, 5, 3, 3))
for i in range(3):
Wcnn[:, :, 0, i] = c[i]
Wcnn[:, :, 1, i] = c[i]
Wcnn[:, :, 2, i] = c[i]
with tf.variable_scope('noise'):
conv = tf.nn.conv2d(self.noise,
Wcnn, [1, 1, 1, 1],
padding='SAME',
name='srm')
self._layers['noise'] = conv
with slim.arg_scope(resnet_arg_scope(is_training=is_training)):
noise_net = resnet_utils.conv2d_same(conv,
64,
7,
stride=2,
scope='conv1')
noise_net = tf.pad(noise_net, [[0, 0], [1, 1], [1, 1], [0, 0]])
noise_net = slim.max_pool2d(noise_net, [3, 3],
stride=2,
padding='VALID',
scope='pool1')
noise_conv4, _ = resnet_v1.resnet_v1(noise_net,
blocks[0:-1],
global_pool=False,
include_root_block=False,
scope='noise')
with tf.variable_scope(self._resnet_scope, self._resnet_scope):
# build the anchors for the image
self._anchor_component()
# rpn
rpn = slim.conv2d(net_conv4,
512, [3, 3],
trainable=is_training,
weights_initializer=initializer,
scope="rpn_conv/3x3")
self._act_summaries.append(rpn)
rpn_cls_score = slim.conv2d(rpn,
self._num_anchors * 2, [1, 1],
trainable=is_training,
weights_initializer=initializer,
padding='VALID',
activation_fn=None,
scope='rpn_cls_score')
# change it so that the score has 2 as its channel size
rpn_cls_score_reshape = self._reshape_layer(
rpn_cls_score, 2, 'rpn_cls_score_reshape')
rpn_cls_prob_reshape = self._softmax_layer(rpn_cls_score_reshape,
"rpn_cls_prob_reshape")
rpn_cls_prob = self._reshape_layer(rpn_cls_prob_reshape,
self._num_anchors * 2,
"rpn_cls_prob")
rpn_bbox_pred = slim.conv2d(rpn,
self._num_anchors * 4, [1, 1],
trainable=is_training,
weights_initializer=initializer,
padding='VALID',
activation_fn=None,
scope='rpn_bbox_pred')
if is_training:
rois, roi_scores = self._proposal_layer(
rpn_cls_prob, rpn_bbox_pred, "rois")
rpn_labels = self._anchor_target_layer(rpn_cls_score, "anchor")
# Try to have a determinestic order for the computing graph, for reproducibility
with tf.control_dependencies([rpn_labels]):
rois, _ = self._proposal_target_layer(
rois, roi_scores, "rpn_rois")
else:
if cfg.TEST.MODE == 'nms':
rois, _ = self._proposal_layer(rpn_cls_prob, rpn_bbox_pred,
"rois")
elif cfg.TEST.MODE == 'top':
rois, _ = self._proposal_top_layer(rpn_cls_prob,
rpn_bbox_pred, "rois")
else:
raise NotImplementedError
# rcnn
if cfg.POOLING_MODE == 'crop':
pool5 = self._crop_pool_layer(net_conv4, rois, "pool5")
else:
raise NotImplementedError
noise_pool5 = self._crop_pool_layer(noise_conv4, rois, "noise_pool5")
with slim.arg_scope(resnet_arg_scope(is_training=is_training)):
noise_fc7, _ = resnet_v1.resnet_v1(noise_pool5,
blocks[-1:],
global_pool=False,
include_root_block=False,
scope='noise')
with slim.arg_scope(resnet_arg_scope(is_training=is_training)):
fc7, _ = resnet_v1.resnet_v1(pool5,
blocks[-1:],
global_pool=False,
include_root_block=False,
scope=self._resnet_scope)
self._layers['fc7'] = fc7
with tf.variable_scope('noise_pred'):
bilinear_pool = compact_bilinear_pooling_layer(fc7,
noise_fc7,
2048 * 8,
compute_size=16,
sequential=False)
fc7 = tf.Print(fc7, [tf.shape(fc7)],
message='Value of %s' % 'fc',
summarize=4,
first_n=1)
bilinear_pool = tf.reshape(bilinear_pool, [-1, 2048 * 8])
bilinear_pool = tf.Print(bilinear_pool, [tf.shape(bilinear_pool)],
message='Value of %s' % 'Blinear',
summarize=4,
first_n=1)
bilinear_pool = tf.multiply(tf.sign(bilinear_pool),
tf.sqrt(tf.abs(bilinear_pool) + 1e-12))
bilinear_pool = tf.nn.l2_normalize(bilinear_pool, dim=1)
noise_cls_score = slim.fully_connected(
bilinear_pool,
self._num_classes,
weights_initializer=tf.contrib.layers.xavier_initializer(),
trainable=is_training,
activation_fn=None,
scope='cls_score')
cls_prob = self._softmax_layer(noise_cls_score, "cls_prob")
fc7 = tf.reduce_mean(fc7, axis=[1, 2])
bbox_pred = slim.fully_connected(
fc7,
self._num_classes * 4,
weights_initializer=initializer_bbox,
trainable=is_training,
activation_fn=None,
scope='bbox_pred')
self._predictions["rpn_cls_score"] = rpn_cls_score
self._predictions["rpn_cls_score_reshape"] = rpn_cls_score_reshape
self._predictions["rpn_cls_prob"] = rpn_cls_prob
self._predictions["rpn_bbox_pred"] = rpn_bbox_pred
self._predictions["cls_score"] = noise_cls_score
self._predictions["cls_prob"] = cls_prob
self._predictions["bbox_pred"] = bbox_pred
self._predictions["rois"] = rois
self._score_summaries.update(self._predictions)
return rois, cls_prob, bbox_pred
def ten_regions(logits):
return tf.map_fn(_ten_regions,tf.abs(logits),dtype=tf.float32)
def _identity_loss(self, real_img, same_img):
return 0.5 * self.LAMBDA * tf.reduce_mean(tf.abs(real_img - same_img), axis=(1, 2, 3))
def Huber_loss(x,y):
return tf.reduce_mean(tf.where(tf.less_equal(tf.abs(x-y), 1.), tf.square(x-y)/2, tf.abs(x-y)-1/2))
def main(args):
network = importlib.import_module(args.model_def)
image_size = (args.image_size, args.image_size)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
stat_file_name = os.path.join(log_dir, 'stat.h5')
# Write arguments to a text file
facenet.write_arguments_to_file(args, os.path.join(log_dir,
'arguments.txt'))
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
random.seed(args.seed)
dataset = facenet.get_dataset(args.data_dir)
if args.filter_filename:
dataset = filter_dataset(dataset,
os.path.expanduser(args.filter_filename),
args.filter_percentile,
args.filter_min_nrof_images_per_class)
if args.validation_set_split_ratio > 0.0:
train_set, val_set = facenet.split_dataset(
dataset, args.validation_set_split_ratio,
args.min_nrof_val_images_per_class, 'SPLIT_IMAGES')
else:
train_set, val_set = dataset, []
nrof_classes = len(train_set)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
pretrained_model = None
if args.pretrained_model:
pretrained_model = os.path.expanduser(args.pretrained_model)
print('Pre-trained model: %s' % pretrained_model)
if args.lfw_dir:
print('LFW directory: %s' % args.lfw_dir)
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs))
# Get the paths for the corresponding images
lfw_paths, actual_issame = lfw.get_paths(
os.path.expanduser(args.lfw_dir), pairs)
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False)
# Get a list of image paths and their labels
image_list, label_list = facenet.get_image_paths_and_labels(train_set)
assert len(image_list) > 0, 'The training set should not be empty'
val_image_list, val_label_list = facenet.get_image_paths_and_labels(
val_set)
# Create a queue that produces indices into the image_list and label_list
labels = ops.convert_to_tensor(label_list, dtype=tf.int32)
range_size = array_ops.shape(labels)[0]
index_queue = tf.train.range_input_producer(range_size,
num_epochs=None,
shuffle=True,
seed=None,
capacity=32)
index_dequeue_op = index_queue.dequeue_many(
args.batch_size * args.epoch_size, 'index_dequeue')
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
batch_size_placeholder = tf.placeholder(tf.int32, name='batch_size')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
image_paths_placeholder = tf.placeholder(tf.string,
shape=(None, 1),
name='image_paths')
labels_placeholder = tf.placeholder(tf.int32,
shape=(None, 1),
name='labels')
control_placeholder = tf.placeholder(tf.int32,
shape=(None, 1),
name='control')
nrof_preprocess_threads = 4
input_queue = data_flow_ops.FIFOQueue(
capacity=2000000,
dtypes=[tf.string, tf.int32, tf.int32],
shapes=[(1, ), (1, ), (1, )],
shared_name=None,
name=None)
enqueue_op = input_queue.enqueue_many(
[image_paths_placeholder, labels_placeholder, control_placeholder],
name='enqueue_op')
image_batch, label_batch = facenet.create_input_pipeline(
input_queue, image_size, nrof_preprocess_threads,
batch_size_placeholder)
image_batch = tf.identity(image_batch, 'image_batch')
image_batch = tf.identity(image_batch, 'input')
label_batch = tf.identity(label_batch, 'label_batch')
print('Number of classes in training set: %d' % nrof_classes)
print('Number of examples in training set: %d' % len(image_list))
print('Number of classes in validation set: %d' % len(val_set))
print('Number of examples in validation set: %d' % len(val_image_list))
print('Building training graph')
# Build the inference graph
prelogits, _ = network.inference(
image_batch,
args.keep_probability,
phase_train=phase_train_placeholder,
bottleneck_layer_size=args.embedding_size,
weight_decay=args.weight_decay)
logits = slim.fully_connected(
prelogits,
len(train_set),
activation_fn=None,
weights_initializer=slim.initializers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(args.weight_decay),
scope='Logits',
reuse=False)
embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings')
# Norm for the prelogits
eps = 1e-4
prelogits_norm = tf.reduce_mean(
tf.norm(tf.abs(prelogits) + eps, ord=args.prelogits_norm_p,
axis=1))
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES,
prelogits_norm * args.prelogits_norm_loss_factor)
# Add center loss
prelogits_center_loss, _ = facenet.center_loss(prelogits, label_batch,
args.center_loss_alfa,
nrof_classes)
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES,
prelogits_center_loss * args.center_loss_factor)
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# Calculate the average cross entropy loss across the batch
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_batch,
logits=logits,
name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
correct_prediction = tf.cast(
tf.equal(tf.argmax(logits, 1), tf.cast(label_batch, tf.int64)),
tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
# Calculate the total losses
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
total_loss = tf.add_n([cross_entropy_mean] + regularization_losses,
name='total_loss')
# Build a Graph that trains the model with one batch of examples and updates the model parameters
train_op = facenet.train(total_loss, global_step, args.optimizer,
learning_rate, args.moving_average_decay,
tf.global_variables(), args.log_histograms)
# Create a saver
saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_memory_fraction,
allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False))
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
coord = tf.train.Coordinator()
tf.train.start_queue_runners(coord=coord, sess=sess)
with sess.as_default():
if pretrained_model:
print('Restoring pretrained model: %s' % pretrained_model)
saver.restore(sess, pretrained_model)
# Training and validation loop
print('Running training')
nrof_steps = args.max_nrof_epochs * args.epoch_size
nrof_val_samples = int(
math.ceil(args.max_nrof_epochs / args.validate_every_n_epochs)
) # Validate every validate_every_n_epochs as well as in the last epoch
stat = {
'loss':
np.zeros((nrof_steps, ), np.float32),
'center_loss':
np.zeros((nrof_steps, ), np.float32),
'reg_loss':
np.zeros((nrof_steps, ), np.float32),
'xent_loss':
np.zeros((nrof_steps, ), np.float32),
'prelogits_norm':
np.zeros((nrof_steps, ), np.float32),
'accuracy':
np.zeros((nrof_steps, ), np.float32),
'val_loss':
np.zeros((nrof_val_samples, ), np.float32),
'val_xent_loss':
np.zeros((nrof_val_samples, ), np.float32),
'val_accuracy':
np.zeros((nrof_val_samples, ), np.float32),
'lfw_accuracy':
np.zeros((args.max_nrof_epochs, ), np.float32),
'lfw_valrate':
np.zeros((args.max_nrof_epochs, ), np.float32),
'learning_rate':
np.zeros((args.max_nrof_epochs, ), np.float32),
'time_train':
np.zeros((args.max_nrof_epochs, ), np.float32),
'time_validate':
np.zeros((args.max_nrof_epochs, ), np.float32),
'time_evaluate':
np.zeros((args.max_nrof_epochs, ), np.float32),
'prelogits_hist':
np.zeros((args.max_nrof_epochs, 1000), np.float32),
}
for epoch in range(1, args.max_nrof_epochs + 1):
step = sess.run(global_step, feed_dict=None)
# Train for one epoch
t = time.time()
cont = train(
args, sess, epoch, image_list, label_list,
index_dequeue_op, enqueue_op, image_paths_placeholder,
labels_placeholder, learning_rate_placeholder,
phase_train_placeholder, batch_size_placeholder,
control_placeholder, global_step, total_loss, train_op,
summary_op, summary_writer, regularization_losses,
args.learning_rate_schedule_file, stat, cross_entropy_mean,
accuracy, learning_rate, prelogits, prelogits_center_loss,
args.random_rotate, args.random_crop, args.random_flip,
prelogits_norm, args.prelogits_hist_max,
args.use_fixed_image_standardization)
stat['time_train'][epoch - 1] = time.time() - t
if not cont:
break
t = time.time()
if len(val_image_list) > 0 and (
(epoch - 1) % args.validate_every_n_epochs
== args.validate_every_n_epochs - 1
or epoch == args.max_nrof_epochs):
validate(args, sess, epoch, val_image_list, val_label_list,
enqueue_op, image_paths_placeholder,
labels_placeholder, control_placeholder,
phase_train_placeholder, batch_size_placeholder,
stat, total_loss, regularization_losses,
cross_entropy_mean, accuracy,
args.validate_every_n_epochs,
args.use_fixed_image_standardization)
stat['time_validate'][epoch - 1] = time.time() - t
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer,
model_dir, subdir, epoch)
# Evaluate on LFW
t = time.time()
if args.lfw_dir:
evaluate(sess, enqueue_op, image_paths_placeholder,
labels_placeholder, phase_train_placeholder,
batch_size_placeholder, control_placeholder,
embeddings, label_batch, lfw_paths, actual_issame,
args.lfw_batch_size, args.lfw_nrof_folds, log_dir,
step, summary_writer, stat, epoch,
args.lfw_distance_metric, args.lfw_subtract_mean,
args.lfw_use_flipped_images,
args.use_fixed_image_standardization)
stat['time_evaluate'][epoch - 1] = time.time() - t
print('Saving statistics')
with h5py.File(stat_file_name, 'w') as f:
for key, value in stat.iteritems():
f.create_dataset(key, data=value)
return model_dir
def huber_loss(labels, predictions, delta=14.0):
residual = tf.abs(labels - predictions)
def f1(): return 0.5 * tf.square(residual)
def f2(): return delta * residual - 0.5 * tf.square(delta)
return tf.cond(residual < delta, f1, f2)
def add_loss(self):
'''Adds loss to the model. Sets "loss" field. initialize must have been called.'''
hp = self._hparams
self.tower_before_loss = []
self.tower_after_loss= []
self.tower_stop_token_loss = []
self.tower_regularization_loss = []
self.tower_linear_loss = []
self.tower_adversarial_loss = []
self.tower_loss = []
total_before_loss = 0
total_after_loss= 0
total_stop_token_loss = 0
total_regularization_loss = 0
total_linear_loss = 0
total_adversarial_loss = 0
total_loss = 0
gpus = ["/gpu:{}".format(i) for i in range(hp.tacotron_gpu_start_idx, hp.tacotron_gpu_start_idx+hp.tacotron_num_gpus)]
for i in range(hp.tacotron_num_gpus):
with tf.device(tf.train.replica_device_setter(ps_tasks=1,ps_device="/cpu:0",worker_device=gpus[i])):
with tf.variable_scope('loss') as scope:
if hp.mask_decoder:
# Compute loss of predictions before postnet
before = MaskedMSE(self.tower_mel_targets[i], self.tower_decoder_output[i], self.tower_targets_lengths[i],
hparams=self._hparams)
# Compute loss after postnet
after = MaskedMSE(self.tower_mel_targets[i], self.tower_mel_outputs[i], self.tower_targets_lengths[i],
hparams=self._hparams)
#Compute <stop_token> loss (for learning dynamic generation stop)
stop_token_loss = MaskedSigmoidCrossEntropy(self.tower_stop_token_targets[i],
self.tower_stop_token_prediction[i], self.tower_targets_lengths[i], hparams=self._hparams)
#Compute masked linear loss
if hp.predict_linear:
#Compute Linear L1 mask loss (priority to low frequencies)
linear_loss = MaskedLinearLoss(self.tower_linear_targets[i], self.tower_linear_outputs[i],
self.targets_lengths, hparams=self._hparams)
else:
linear_loss=0.
else:
# Compute loss of predictions before postnet
before = tf.losses.mean_squared_error(self.tower_mel_targets[i], self.tower_decoder_output[i])
# Compute loss after postnet
after = tf.losses.mean_squared_error(self.tower_mel_targets[i], self.tower_mel_outputs[i])
#Compute <stop_token> loss (for learning dynamic generation stop)
stop_token_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.tower_stop_token_targets[i],
logits=self.tower_stop_token_prediction[i]))
speaker_loss=tf.losses
if hp.predict_linear:
#Compute linear loss
#From https://github.com/keithito/tacotron/blob/tacotron2-work-in-progress/models/tacotron.py
#Prioritize loss for frequencies under 2000 Hz.
l1 = tf.abs(self.tower_linear_targets[i] - self.tower_linear_outputs[i])
n_priority_freq = int(2000 / (hp.sample_rate * 0.5) * hp.num_freq)
linear_loss = 0.5 * tf.reduce_mean(l1) + 0.5 * tf.reduce_mean(l1[:,:,0:n_priority_freq])
else:
linear_loss = 0.
# Compute the regularization weight
if hp.tacotron_scale_regularization:
reg_weight_scaler = 1. / (2 * hp.max_abs_value) if hp.symmetric_mels else 1. / (hp.max_abs_value)
reg_weight = hp.tacotron_reg_weight * reg_weight_scaler
else:
reg_weight = hp.tacotron_reg_weight
# Regularize variables
# Exclude all types of bias, RNN (Bengio et al. On the difficulty of training recurrent neural networks), embeddings and prediction projection layers.
# Note that we consider attention mechanism v_a weights as a prediction projection layer and we don't regularize it. (This gave better stability)
regularization = tf.add_n([tf.nn.l2_loss(v) for v in self.all_vars
if not('bias' in v.name or 'Bias' in v.name or '_projection' in v.name or 'inputs_embedding' in v.name
or 'RNN' in v.name or 'LSTM' in v.name)]) * reg_weight
# Compute the speaker adversarial training loss
# speaker_prediction: predicted speaker label for each time step of input, with shape [N, T_in, speaker_num]
# speaker_targets: one-hot speaker label of current input, from shape [N, speaker_num] to [N, 1, speaker_num] to [N, T_in, speaker_num]
seq_len = tf.shape(self.tower_predict_speaker_labels[i])[1]
speaker_targets = tf.one_hot(self.tower_speaker_labels[i], hp.speaker_num, dtype=tf.float32)
speaker_targets = tf.tile(tf.reshape(speaker_targets, shape=[-1, 1, hp.speaker_num]),
multiples=[1, seq_len, 1])
adversarial_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels=speaker_targets,
logits=self.tower_predict_speaker_labels[i]))
# Compute final loss term
self.tower_before_loss.append(before)
self.tower_after_loss.append(after)
self.tower_stop_token_loss.append(stop_token_loss)
self.tower_regularization_loss.append(regularization)
self.tower_linear_loss.append(linear_loss)
self.tower_adversarial_loss.append(adversarial_loss)
loss = before + after + stop_token_loss + regularization + linear_loss + hp.loss_weight * adversarial_loss + self.kl_div
self.tower_loss.append(loss)
for i in range(hp.tacotron_num_gpus):
total_before_loss += self.tower_before_loss[i]
total_after_loss += self.tower_after_loss[i]
total_stop_token_loss += self.tower_stop_token_loss[i]
total_regularization_loss += self.tower_regularization_loss[i]
total_linear_loss += self.tower_linear_loss[i]
total_adversarial_loss +=self.tower_adversarial_loss[i]
total_loss += self.tower_loss[i]
self.before_loss = total_before_loss / hp.tacotron_num_gpus
self.after_loss = total_after_loss / hp.tacotron_num_gpus
self.stop_token_loss = total_stop_token_loss / hp.tacotron_num_gpus
self.regularization_loss = total_regularization_loss / hp.tacotron_num_gpus
self.linear_loss = total_linear_loss / hp.tacotron_num_gpus
self.adversarial_loss = total_adversarial_loss / hp.tacotron_num_gpus
self.loss = total_loss / hp.tacotron_num_gpus
def interpolate(times,
interval_values,
interval_times,
validate_args=False,
dtype=None,
name=None):
"""Performs the monotone convex interpolation.
The monotone convex method is a scheme devised by Hagan and West (Ref [1]). It
is a commonly used method to interpolate interest rate yield curves. For
more details see Refs [1, 2].
It is important to point out that the monotone convex method *does not* solve
the standard interpolation problem but a modified one as described below.
Suppose we are given a strictly increasing sequence of scalars (which we will
refer to as time) `[t_1, t_2, ... t_n]` and a set of values
`[f_1, f_2, ... f_n]`.
The aim is to find a function `f(t)` defined on the interval `[0, t_n]` which
satisfies (in addition to continuity and positivity conditions) the following
```None
Integral[f(u), t_{i-1} <= u <= t_i] = f_i, with t_0 = 0
```
In the context of interest rate curve building, `f(t)` corresponds to the
instantaneous forward rate at time `t` and the `f_i` correspond to the
discrete forward rates that apply to the time period `[t_{i-1}, t_i]`.
Furthermore, the integral of the forward curve is related to the yield curve
by
```None
Integral[f(u), 0 <= u <= t] = r(t) * t
```
where `r(t)` is the interest rate that applies between `[0, t]` (the yield of
a zero coupon bond paying a unit of currency at time `t`).
This function computes both the interpolated value and the integral along
the segment containing the supplied time. Specifically, given a time `t` such
that `t_k <= t <= t_{k+1}`, this function computes the interpolated value
`f(t)` and the value `Integral[f(u), t_k <= u <= t]`.
This implementation of the method currently supports batching along the
interpolation times but not along the interpolated curves (i.e. it is possible
to evaluate the `f(t)` for `t` as a vector of times but not build multiple
curves at the same time).
### Example
```python
interval_times = tf.constant([0.25, 0.5, 1.0, 2.0, 3.0], dtype=dtype)
interval_values = tf.constant([0.05, 0.051, 0.052, 0.053, 0.055],
dtype=dtype)
times = tf.constant([0.25, 0.5, 1.0, 2.0, 3.0, 1.1], dtype=dtype)
# Returns the following two values:
# interpolated = [0.0505, 0.05133333, 0.05233333, 0.054, 0.0555, 0.05241]
# integrated = [0, 0, 0, 0, 0.055, 0.005237]
# Note that the first four integrated values are zero. This is because
# those interpolation time are at the start of their containing interval.
# The fourth value (i.e. at 3.0) is not zero because this is the last
# interval (i.e. it is the integral from 2.0 to 3.0).
interpolated, integrated = interpolate(
times, interval_values, interval_times)
```
### References:
[1]: Patrick Hagan & Graeme West. Interpolation Methods for Curve
Construction. Applied Mathematical Finance. Vol 13, No. 2, pp 89-129.
June 2006.
https://www.researchgate.net/publication/24071726_Interpolation_Methods_for_Curve_Construction
[2]: Patrick Hagan & Graeme West. Methods for Constructing a Yield Curve.
Wilmott Magazine, pp. 70-81. May 2008.
Args:
times: Non-negative rank 1 `Tensor` of any size. The times for which the
interpolation has to be performed.
interval_values: Rank 1 `Tensor` of the same shape and dtype as
`interval_times`. The values associated to each of the intervals specified
by the `interval_times`. Must have size at least 2.
interval_times: Strictly positive rank 1 `Tensor` of real dtype containing
increasing values. The endpoints of the intervals (i.e. `t_i` above.).
Note that the left end point of the first interval is implicitly assumed
to be 0. Must have size at least 2.
validate_args: Python bool. If true, adds control dependencies to check that
the `times` are bounded by the `interval_endpoints`.
Default value: False
dtype: `tf.Dtype` to use when converting arguments to `Tensor`s. If not
supplied, the default Tensorflow conversion will take place. Note that
this argument does not do any casting.
Default value: None.
name: Python `str` name prefixed to Ops created by this class.
Default value: None which is mapped to the default name 'interpolation'.
Returns:
A 2-tuple containing
interpolated_values: Rank 1 `Tensor` of the same size and dtype as the
`times`. The interpolated values at the supplied times.
integrated_values: Rank 1 `Tensor` of the same size and dtype as the
`times`. The integral of the interpolated function. The integral is
computed from the largest interval time that is smaller than the time
up to the given time.
"""
with tf.compat.v1.name_scope(
name,
default_name='interpolate',
values=[times, interval_times, interval_values]):
times = tf.convert_to_tensor(times, dtype=dtype, name='times')
interval_times = tf.convert_to_tensor(
interval_times, dtype=dtype, name='interval_times')
interval_values = tf.convert_to_tensor(
interval_values, dtype=dtype, name='interval_values')
control_deps = []
if validate_args:
control_deps = [
tf.compat.v1.debugging.assert_non_negative(times),
tf.compat.v1.debugging.assert_positive(interval_times)
]
with tf.compat.v1.control_dependencies(control_deps):
# Step 1: Find the values at the endpoints.
endpoint_values = _interpolate_adjacent(interval_times, interval_values)
endpoint_times = tf.concat([[0.0], interval_times], axis=0)
intervals = piecewise.find_interval_index(
times, endpoint_times, last_interval_is_closed=True)
# Comparing to the notation used in the paper:
# f_left -> f_{i-1}
# f_right -> f_i
# t_left -> t_{i-1}
# t_right -> t_i
# fd -> f^d_i
# g0 -> g0
# g1 -> g1
# g1plus2g0 -> g1 + 2 g0 (boundary line A)
# g0plus2g1 -> g0 + 2 g1 (boundary line B)
# x -> x
f_left = tf.gather(endpoint_values, intervals)
f_right = tf.gather(endpoint_values, intervals + 1)
# fd is the discrete forward associated to each interval.
fd = tf.gather(interval_values, intervals)
t_left = tf.gather(endpoint_times, intervals)
t_right = tf.gather(endpoint_times, intervals + 1)
interval_lengths = (t_right - t_left)
x = (times - t_left) / interval_lengths
# TODO(b/140410758): The calculation below can be done more efficiently
# if we instead do the following:
# 1. Subdivide the regions further so that each subregion corresponds
# to a single quadratic in x. (Region 2, 3 and 4 get divided into 2
# pieces for a total of 7 cases.
# 2. For each interval (i.e. [t_i, t{i+1}]) the case that applies to
# a point falling in that region can be decided and the corresponding
# quadratic coefficients computed once and for all.
# 3. The above information is built once for the supplied forwards.
# 4. Given the above information and a set of times to interpolate for,
# we map each time to the appropriate interval and compute the quadratic
# function value using that x.
g0 = f_left - fd
g1 = f_right - fd
g1plus2g0 = g1 + 2 * g0
g0plus2g1 = g0 + 2 * g1
result = tf.zeros_like(times)
integrated = tf.zeros_like(times)
# The method uses quadratic splines to do the interpolation.
# The specific spline used depends on the relationship between the
# boundary values (`g0` and `g1` above).
# The two dimensional plane determined by these two values is divided
# into four wedge sections referred to as region 1, 2, 3 and 4 below.
# For details of how the regions are defined, see Fig. 4 in Ref [2].
is_region_1, region_1_value, integrated_value_1 = _region_1(
g1plus2g0, g0plus2g1, g0, g1, x)
result = tf.where(is_region_1, region_1_value, result)
integrated = tf.where(is_region_1, integrated_value_1, integrated)
is_region_2, region_2_value, integrated_value_2 = _region_2(
g1plus2g0, g0plus2g1, g0, g1, x)
result = tf.where(is_region_2, region_2_value, result)
integrated = tf.where(is_region_2, integrated_value_2, integrated)
is_region_3, region_3_value, integrated_value_3 = _region_3(
g1plus2g0, g0plus2g1, g0, g1, x)
result = tf.where(is_region_3, region_3_value, result)
integrated = tf.where(is_region_3, integrated_value_3, integrated)
is_region_4, region_4_value, integrated_value_4 = _region_4(
g1plus2g0, g0plus2g1, g0, g1, x)
result = tf.where(is_region_4, region_4_value, result)
integrated = tf.where(is_region_4, integrated_value_4, integrated)
# g0 = g1 = 0 requires special handling. Checking if the values are
# legitimatey zero requires we pay close attention to the numerical
# precision issues.
g0_eps = tf.abs(tf.math.nextafter(fd, f_left) - fd) * 1.1
g1_eps = tf.abs(tf.math.nextafter(fd, f_right) - fd) * 1.1
is_origin = ((tf.abs(g0) <= g0_eps) & (tf.abs(g1) <= g1_eps))
result = tf.where(is_origin, tf.zeros_like(result), result)
integrated = tf.where(is_origin, tf.zeros_like(integrated), integrated)
return (result + fd, (integrated + fd * x) * interval_lengths)
def conv(a: AbsArrow, args: TensorVarList, state) -> Sequence[Tensor]:
return [tf.abs(args[0])]
def build_loss(self):
"""Adds ops for computing loss."""
with tf.name_scope('compute_loss'):
self.reconstr_loss = 0
self.smooth_loss = 0
self.ssim_loss = 0
self.icp_transform_loss = 0
self.icp_residual_loss = 0
# self.images is organized by ...[scale][B, h, w, seq_len * 3].
self.images = [None for _ in range(NUM_SCALES)]
# Following nested lists are organized by ...[scale][source-target].
self.warped_image = [{} for _ in range(NUM_SCALES)]
self.warp_mask = [{} for _ in range(NUM_SCALES)]
self.warp_error = [{} for _ in range(NUM_SCALES)]
self.ssim_error = [{} for _ in range(NUM_SCALES)]
self.icp_transform = [{} for _ in range(NUM_SCALES)]
self.icp_residual = [{} for _ in range(NUM_SCALES)]
self.middle_frame_index = util.get_seq_middle(self.seq_length)
# Compute losses at each scale.
for s in range(NUM_SCALES):
# Scale image stack.
if s == 0: # Just as a precaution. TF often has interpolation bugs.
self.images[s] = self.image_stack
else:
height_s = int(self.img_height / (2**s))
width_s = int(self.img_width / (2**s))
self.images[s] = tf.image.resize_bilinear(
self.image_stack, [height_s, width_s],
align_corners=True)
# Smoothness.
if self.smooth_weight > 0:
for i in range(self.seq_length):
# When computing minimum loss, use the depth map from the middle
# frame only.
if not self.compute_minimum_loss or i == self.middle_frame_index:
disp_smoothing = self.disp[i][s]
if self.depth_normalization:
# Perform depth normalization, dividing by the mean.
mean_disp = tf.reduce_mean(disp_smoothing,
axis=[1, 2, 3],
keep_dims=True)
disp_input = disp_smoothing / mean_disp
else:
disp_input = disp_smoothing
scaling_f = (1.0 if self.equal_weighting else 1.0 /
(2**s))
self.smooth_loss += scaling_f * self.depth_smoothness(
disp_input, self.images[s][:, :, :, 3 * i:3 *
(i + 1)])
self.debug_all_warped_image_batches = []
for i in range(self.seq_length):
for j in range(self.seq_length):
if i == j:
continue
# When computing minimum loss, only consider the middle frame as
# target.
if self.compute_minimum_loss and j != self.middle_frame_index:
continue
# We only consider adjacent frames, unless either
# compute_minimum_loss is on (where the middle frame is matched with
# all other frames) or exhaustive_mode is on (where all frames are
# matched with each other).
if (not self.compute_minimum_loss
and not self.exhaustive_mode
and abs(i - j) != 1):
continue
selected_scale = 0 if self.depth_upsampling else s
source = self.images[selected_scale][:, :, :,
3 * i:3 * (i + 1)]
target = self.images[selected_scale][:, :, :,
3 * j:3 * (j + 1)]
if self.depth_upsampling:
target_depth = self.depth_upsampled[j][s]
else:
target_depth = self.depth[j][s]
key = '%d-%d' % (i, j)
if self.handle_motion:
# self.seg_stack of shape (B, H, W, 9).
# target_depth corresponds to middle frame, of shape (B, H, W, 1).
# Now incorporate the other warping results, performed according
# to the object motion network's predictions.
# self.object_masks batch_size elements of (N, H, W, 9).
# self.object_masks_warped batch_size elements of (N, H, W, 9).
# self.object_transforms batch_size elements of (N, 2, 6).
self.all_batches = []
for batch_s in range(self.batch_size):
# To warp i into j, first take the base warping (this is the
# full image i warped into j using only the egomotion estimate).
base_warping = self.warped_seq[s][i][batch_s]
transform_matrices_thisbatch = tf.map_fn(
lambda transform: project.
get_transform_mat(
tf.expand_dims(transform, axis=0), i, j
)[0], self.object_transforms[0][batch_s])
def inverse_warp_wrapper(matrix):
"""Wrapper for inverse warping method."""
warp_image, _ = (project.inverse_warp(
tf.expand_dims(base_warping, axis=0),
tf.expand_dims(target_depth[batch_s],
axis=0),
tf.expand_dims(matrix, axis=0),
tf.expand_dims(self.intrinsic_mat[
batch_s, selected_scale, :, :],
axis=0),
tf.expand_dims(self.intrinsic_mat_inv[
batch_s, selected_scale, :, :],
axis=0)))
return warp_image
warped_images_thisbatch = tf.map_fn(
inverse_warp_wrapper,
transform_matrices_thisbatch,
dtype=tf.float32)
warped_images_thisbatch = warped_images_thisbatch[:,
0, :, :, :]
# warped_images_thisbatch is now of shape (N, H, W, 9).
# Combine warped frames into a single one, using the object
# masks. Result should be (1, 128, 416, 3).
# Essentially, we here want to sum them all up, filtered by the
# respective object masks.
mask_base_valid_source = tf.equal(
self.seg_stack[batch_s, :, :,
i * 3:(i + 1) * 3],
tf.constant(0, dtype=tf.uint8))
mask_base_valid_target = tf.equal(
self.seg_stack[batch_s, :, :,
j * 3:(j + 1) * 3],
tf.constant(0, dtype=tf.uint8))
mask_valid = tf.logical_and(
mask_base_valid_source,
mask_base_valid_target)
self.base_warping = base_warping * tf.to_float(
mask_valid)
background = tf.expand_dims(self.base_warping,
axis=0)
def construct_const_filter_tensor(obj_id):
return tf.fill(
dims=[
self.img_height, self.img_width, 3
],
value=tf.sign(obj_id)) * tf.to_float(
tf.equal(
self.seg_stack[batch_s, :, :,
3:6],
tf.cast(obj_id,
dtype=tf.uint8)))
filter_tensor = tf.map_fn(
construct_const_filter_tensor,
tf.to_float(self.object_ids[s][batch_s]))
filter_tensor = tf.stack(filter_tensor, axis=0)
objects_to_add = tf.reduce_sum(tf.multiply(
warped_images_thisbatch, filter_tensor),
axis=0,
keepdims=True)
combined = background + objects_to_add
self.all_batches.append(combined)
# Now of shape (B, 128, 416, 3).
self.warped_image[s][key] = tf.concat(
self.all_batches, axis=0)
else:
# Don't handle motion, classic model formulation.
egomotion_mat_i_j = project.get_transform_mat(
self.egomotion, i, j)
# Inverse warp the source image to the target image frame for
# photometric consistency loss.
self.warped_image[s][key], self.warp_mask[s][
key] = (project.inverse_warp(
source, target_depth, egomotion_mat_i_j,
self.intrinsic_mat[:,
selected_scale, :, :],
self.
intrinsic_mat_inv[:,
selected_scale, :, :]))
# Reconstruction loss.
self.warp_error[s][key] = tf.abs(
self.warped_image[s][key] - target)
if not self.compute_minimum_loss:
self.reconstr_loss += tf.reduce_mean(
self.warp_error[s][key] *
self.warp_mask[s][key])
# SSIM.
if self.ssim_weight > 0:
self.ssim_error[s][key] = self.ssim(
self.warped_image[s][key], target)
# TODO(rezama): This should be min_pool2d().
if not self.compute_minimum_loss:
ssim_mask = slim.avg_pool2d(
self.warp_mask[s][key], 3, 1, 'VALID')
self.ssim_loss += tf.reduce_mean(
self.ssim_error[s][key] * ssim_mask)
# If the minimum loss should be computed, the loss calculation has been
# postponed until here.
if self.compute_minimum_loss:
for frame_index in range(self.middle_frame_index):
key1 = '%d-%d' % (frame_index, self.middle_frame_index)
key2 = '%d-%d' % (self.seq_length - frame_index - 1,
self.middle_frame_index)
logging.info('computing min error between %s and %s',
key1, key2)
min_error = tf.minimum(self.warp_error[s][key1],
self.warp_error[s][key2])
self.reconstr_loss += tf.reduce_mean(min_error)
if self.ssim_weight > 0: # Also compute the minimum SSIM loss.
min_error_ssim = tf.minimum(
self.ssim_error[s][key1],
self.ssim_error[s][key2])
self.ssim_loss += tf.reduce_mean(min_error_ssim)
# Build the total loss as composed of L1 reconstruction, SSIM, smoothing
# and object size constraint loss as appropriate.
self.reconstr_loss *= self.reconstr_weight
self.total_loss = self.reconstr_loss
if self.smooth_weight > 0:
self.smooth_loss *= self.smooth_weight
self.total_loss += self.smooth_loss
if self.ssim_weight > 0:
self.ssim_loss *= self.ssim_weight
self.total_loss += self.ssim_loss
if self.size_constraint_weight > 0:
self.inf_loss *= self.size_constraint_weight
self.total_loss += self.inf_loss
conductivity_1 = 16
conductivity_2 = 56
heat_filter_1 = conductivity_1 / 3. * np.asarray([[1., 1., 1.], [1., -8., 1.], [1., 1., 1.]]).reshape(3,3,1,1)
heat_filter_2 = conductivity_2 / 3. * np.asarray([[1., 1., 1.], [1., -8., 1.], [1., 1., 1.]]).reshape(3,3,1,1)
################
import scipy.io as sio
u1 = sio.loadmat('/home/hope-yao/Downloads/solution_6666.mat')['U1'][0][1:-1,1:-1]
f1 = sio.loadmat('/home/hope-yao/Downloads/q_6666.mat')['F1'][0][1:-1,1:-1]
f_input = tf.placeholder(tf.float32, shape=(1, 64, 64, 1))
u_opt = tf.Variable(tf.zeros((1,64,64,1)))
padded_input = tf.pad(u_opt, [[0, 0], [1, 1], [1, 1], [0, 0]], "SYMMETRIC") # convolution with symmetric padding at boundary
f_pred_opt = tf.nn.conv2d(input=padded_input, filter=heat_filter_1, strides=[1, 1, 1, 1], padding='VALID')
loss = tf.reduce_mean(tf.abs(f_pred_opt - f_input))
opt = tf.train.AdamOptimizer(0.1)
grads_g = opt.compute_gradients(loss)
apply_gradient_op = opt.apply_gradients(grads_g)
init_op = tf.initialize_variables(tf.all_variables())
## training starts ###
FLAGS = tf.app.flags.FLAGS
tfconfig = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True,
)
tfconfig.gpu_options.allow_growth = True
sess = tf.Session(config=tfconfig)
init = tf.global_variables_initializer()
sess.run(init)
def factorize(A, hyperparameters):
#l1_regularizer_parameter = hyperparameters["l1_regularizer_parameter"]
#dimension = hyperparameters["dimension"]
#zero_out_threshold = hyperparameters["zero_out_threshold"]
#lr = hyperparameters["lr"]
#niters = hyperparameters["niters"]
l1_regularizer_parameter = .01
zero_out_thresholds = [1e-3, 1e-3]
lr = 5e-2
niters = 10000
intermediate_dimension = 200
# Factorize A in to matrices of shape shapes
tf.reset_default_graph()
shapes = [(A.shape[0], intermediate_dimension),
(intermediate_dimension, A.shape[1])]
variables = []
for shape in shapes:
variables.append(
tf.Variable(
tf.random_normal(shape,
stddev=.1 + tf.eye(*shape),
dtype=tf.float32)))
# Multiply the variables together
to_optimize = variables[0] + tf.eye(
*tuple(variables[0].get_shape().as_list()))
for variable in variables[1:]:
to_optimize = tf.matmul(
to_optimize,
tf.eye(*tuple(variable.get_shape().as_list())) + variable)
#to_optimize = tf.matmul(to_optimize, variable)
assert (tuple(to_optimize.get_shape().as_list()) == tuple(A.shape))
# Construct the optimization
target_placeholder = tf.placeholder(tf.float32, shape=A.shape)
loss_frobenius_error = tf.norm(target_placeholder - to_optimize)
# Add l1 loss
l1_parameter_placeholder = tf.placeholder(dtype=tf.float32)
l1_regularizer = tf.contrib.layers.l1_regularizer(scale=1.0, scope=None)
regularization_penalty = tf.contrib.layers.apply_regularization(
l1_regularizer, variables)
loss = loss_frobenius_error + l1_parameter_placeholder * regularization_penalty
#loss = loss_frobenius_error
# Create opt
lr_placeholder = tf.placeholder(dtype=tf.float32)
opt = tf.train.GradientDescentOptimizer(learning_rate=lr_placeholder)
minimize = opt.minimize(loss)
# Zero out values below absolute threshold
zero_ops = []
with tf.control_dependencies([minimize]):
for matrix, thresh in zip(variables, zero_out_thresholds):
mask = tf.cast(
tf.greater(tf.abs(matrix),
thresh * tf.ones_like(matrix, dtype=tf.float32)),
tf.float32)
zero_ops.append(tf.assign(matrix, tf.multiply(mask, matrix)))
minimize_and_zero_out = tf.group(zero_ops)
# Do optimization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
results = []
for i in range(niters):
_, loss_materialized, loss_frobenius_error_materialized = sess.run(
[minimize_and_zero_out, loss, loss_frobenius_error],
feed_dict={
target_placeholder: A,
l1_parameter_placeholder: l1_regularizer_parameter,
lr_placeholder: lr
})
if i % 100 == 0:
# Also calculate sparsity
variables_materialized = sess.run(variables)
total_number_of_nnzs = sum(
[np.count_nonzero(x) for x in variables_materialized])
nnzs = [np.count_nonzero(x) for x in variables_materialized]
print(
"Loss: %g, Loss_frob_error: %g, # nnzs in factored matrices: %d, frob_error / (||u||||v||): %g, nnzs: %s"
% (loss_materialized, loss_frobenius_error_materialized,
total_number_of_nnzs, loss_frobenius_error_materialized /
(np.linalg.norm(u) * np.linalg.norm(v)), str(nnzs)))
results.append(
(total_number_of_nnzs, loss_frobenius_error_materialized))
all_results = {
"hyperparameters": hyperparameters,
"target_matrix": A,
"results": results
}
vs = sess.run(variables)
print([np.count_nonzero(v) for v in vs])
return [(v + np.eye(*v.shape), np.count_nonzero(v))
for v in vs], all_results
def build_model(self):
self.is_training = tf.compat.v1.placeholder(tf.bool,
name='is_training')
self.images = tf.compat.v1.placeholder(tf.float32,
[None] + self.image_shape,
name='real_images')
self.z = tf.compat.v1.placeholder(tf.float32, [None, self.z_dim],
name='z')
self.z_sum = tf.compat.v1.summary.histogram("z", self.z)
self.G = self.generator(self.z)
self.D, self.D_logits = self.discriminator(self.images)
self.D_, self.D_logits_ = self.discriminator(self.G, reuse=True)
self.d_sum = tf.compat.v1.summary.histogram("d", self.D)
self.d__sum = tf.compat.v1.summary.histogram("d_", self.D_)
self.G_sum = tf.compat.v1.summary.image("G", self.G)
self.d_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits,
labels=tf.ones_like(
self.D)))
self.d_loss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_,
labels=tf.zeros_like(
self.D_)))
self.g_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_,
labels=tf.ones_like(
self.D_)))
self.d_loss_real_sum = tf.compat.v1.summary.scalar(
"d_loss_real", self.d_loss_real)
self.d_loss_fake_sum = tf.compat.v1.summary.scalar(
"d_loss_fake", self.d_loss_fake)
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss_sum = tf.compat.v1.summary.scalar("g_loss", self.g_loss)
self.d_loss_sum = tf.compat.v1.summary.scalar("d_loss", self.d_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.saver = tf.compat.v1.train.Saver(max_to_keep=1)
# Completion.
self.mask = tf.placeholder(tf.float32, self.image_shape, name='mask')
self.contextual_loss = tf.reduce_sum(
tf.contrib.layers.flatten(
tf.abs(
tf.multiply(self.mask, self.G) -
tf.multiply(self.mask, self.images))), 1)
self.perceptual_loss = self.g_loss
self.complete_loss = self.contextual_loss + self.lam * self.perceptual_loss
self.grad_complete_loss = tf.gradients(self.complete_loss, self.z)
def _cycle_loss(self, real_img, cycled_img):
return self.LAMBDA * tf.reduce_mean(tf.abs(real_img - cycled_img), axis=(1, 2, 3))
g = tf.Variable(g0)
HS_test = Hypersurface(Z, f, 10000)
HS_test.set_k(k)
test_old = 10
epochs = range(n_epochs)
for epoch in epochs:
train(HS, g, learning_rate, factor)
if epoch % 20 == 0:
h = tf.matmul(g, g, adjoint_b=True)
h = h / h[0][0]
integration = HS.integrate(
lambda patch: tf.abs(patch.num_eta_tf(h) / factor - 1),
tensor=True).numpy()
test = HS_test.integrate(
lambda patch: tf.abs(patch.num_eta_tf(h) / factor - 1),
tensor=True).numpy()
print('train:', integration)
print('test:', test)
if test > test_old:
break
test_old = test
h_minimal = tf.matmul(g, g, adjoint_b=True)
h_minimal = h_minimal / h_minimal[0][0]
train_sigma = HS.integrate(
lambda patch: tf.abs(patch.num_eta_tf(h_minimal) / factor - 1),
def calc(self, input, target, mask=None, is_mask=False):
""" Args:
input [batch_num, class_num]:
The direct prediction of classification fc layer.
target [batch_num, class_num]:
Binary target (0 or 1) for each sample each class. The value is -1
when the sample is ignored.
mask [batch_num, class_num]
"""
edges_left, edges_right = self.edges_left, self.edges_right
mmt = self.momentum
# gradient length
self.g = tf.abs(tf.sigmoid(input) - target) # [batch_num, class_num]
g = tf.expand_dims(self.g, axis=0) # [1, batch_num, class_num]
g_greater_equal_edges_left = tf.greater_equal(
g, edges_left) # [bins, batch_num, class_num]
g_less_edges_right = tf.less(
g, edges_right) # [bins, batch_num, class_num]
zero_matrix = tf.cast(tf.zeros_like(g_greater_equal_edges_left),
dtype=tf.float32) # [bins, batch_num, class_num]
if is_mask:
mask_greater_zero = tf.greater(mask, 0)
inds = tf.cast(tf.logical_and(
tf.logical_and(g_greater_equal_edges_left, g_less_edges_right),
mask_greater_zero),
dtype=tf.float32) # [bins, batch_num, class_num]
tot = tf.maximum(
tf.reduce_sum(tf.cast(mask_greater_zero, dtype=tf.float32)),
1.0)
else:
inds = tf.cast(tf.logical_and(g_greater_equal_edges_left,
g_less_edges_right),
dtype=tf.float32) # [bins, batch_num, class_num]
input_shape = tf.shape(input)
tot = tf.maximum(
tf.cast(input_shape[0] * input_shape[1], dtype=tf.float32),
1.0)
num_in_bin = tf.reduce_sum(inds, axis=[1, 2]) # [bins]
num_in_bin_greater_zero = tf.greater(num_in_bin, 0) # [bins]
num_valid_bin = tf.reduce_sum(
tf.cast(num_in_bin_greater_zero, dtype=tf.float32))
# num_in_bin = num_in_bin + 1e-12
if mmt > 0:
update = tf.assign(self.acc_sum, tf.where(num_in_bin_greater_zero, mmt * self.acc_sum \
+ (1 - mmt) * num_in_bin, self.acc_sum))
with tf.control_dependencies([update]):
self.acc_sum_tmp = tf.identity(self.acc_sum,
name='updated_accsum')
acc_sum = tf.expand_dims(self.acc_sum_tmp, -1) # [bins, 1]
acc_sum = tf.expand_dims(acc_sum, -1) # [bins, 1, 1]
acc_sum = acc_sum + zero_matrix # [bins, batch_num, class_num]
weights = tf.where(tf.equal(inds, 1), tot / acc_sum,
zero_matrix)
weights = tf.reduce_sum(weights, axis=0)
else:
num_in_bin = tf.expand_dims(num_in_bin, -1) # [bins, 1]
num_in_bin = tf.expand_dims(num_in_bin, -1) # [bins, 1, 1]
num_in_bin = num_in_bin + zero_matrix # [bins, batch_num, class_num]
weights = tf.where(tf.equal(inds, 1), tot / num_in_bin,
zero_matrix)
weights = tf.reduce_sum(weights, axis=0)
weights = weights / num_valid_bin
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=target,
logits=input)
loss = tf.reduce_sum(loss * weights) / tot
return loss
def posdef_eig_self_adjoint(mat):
"""Computes eigendecomposition using self_adjoint_eig."""
evals, evecs = tf.self_adjoint_eig(mat)
evals = tf.abs(evals) # Should be equivalent to svd approach.
return evals, evecs
phase = [
tf.constant(np.random.random(size=(size,size)),dtype=tf.float32),
tf.constant(no.random.random(size=(size,size)),dtype=tf.float32)
]
with tf.variable_scope('FullyConnected'):
layer_1 = add_layer_amp(tf.cast(data_x,dtype=tf.complex64),amp[0],phase[0],size,delta)
layer_2 = add_layer_amp(layer_1,amp[1],phase[1],size,delta)
layer_3 = add_layer_amp(layer_2,amp[2],phase[1],size,delta)
layer_4 = add_layer_amp(layer_3,amp[3],phase[1],size,delta)
layer_5 = add_layer_amp(layer_4,amp[4],phase[1],size,delta)
output_layer = add_layer_amp(layer_5,amp[5],phase[1],size,delta)
output = _propogation(output_layer)
with tf.variable_scope('Loss'):
logits_abs = tf.square(tf.nn.softmax(ten_regions(tf.abs(output))))
loss = tf.reduce_sum(tf.square(logits_abs-data_y))
train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
with tf.variable_scope('Accuracy'):
pre_correct = tf.equal(tf.argmax(data_y,1),tf.argmax(logits_abs,1))
accuracy = tf.reduce_mean(tf.cast(pre_correct,tf.float32))
init = tf.global_variables_initializer()
train_epochs = 20
test_epochs = 5
session = tf.Session()
with tf.device('/gpu:0'):
session.run(init)
total_batch = int(mnist.train.num_examples / batch_size)
def absolute_residual(Psi_y, Psi_x, Psi_xx, u=0.01, v=1.0):
return tf.abs(Psi_y + u * Psi_x - v * Psi_xx)
def multihead_attention(emb,
queries,
keys,
num_units=None,
num_heads=8,
dropout_rate=0,
is_training=True,
causality=False,
scope="multihead_attention",
reuse=None):
'''Applies multihead attention.
Args:
queries: A 3d tensor with shape of [N, T_q, C_q].
keys: A 3d tensor with shape of [N, T_k, C_k].
num_units: A scalar. Attention size.
dropout_rate: A floating point number.
is_training: Boolean. Controller of mechanism for dropout.
causality: Boolean. If true, units that reference the future are masked.
num_heads: An int. Number of heads.
scope: Optional scope for `variable_scope`.
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns
A 3d tensor with shape of (N, T_q, C)
'''
with tf.variable_scope(scope, reuse=reuse):
# Set the fall back option for num_units
if num_units is None:
num_units = queries.get_shape().as_list[-1]
# Linear projections
Q = tf.layers.dense(queries, num_units,
activation=tf.nn.relu) # (N, T_q, C)
K = tf.layers.dense(keys, num_units,
activation=tf.nn.relu) # (N, T_k, C)
V = tf.layers.dense(keys, num_units,
activation=tf.nn.relu) # (N, T_k, C)
# Split and concat
Q_ = tf.concat(tf.split(Q, num_heads, axis=2),
axis=0) # (h*N, T_q, C/h)
K_ = tf.concat(tf.split(K, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
V_ = tf.concat(tf.split(V, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
# Multiplication
outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
# Scale
outputs = outputs / (K_.get_shape().as_list()[-1]**0.5)
# Key Masking
key_masks = tf.sign(tf.abs(tf.reduce_sum(emb, axis=-1))) # (N, T_k)
key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
key_masks = tf.tile(tf.expand_dims(key_masks, 1),
[1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(outputs) * (-2**32 + 1)
outputs = tf.where(tf.equal(key_masks, 0), paddings,
outputs) # (h*N, T_q, T_k)
# Causality = Future blinding
if causality:
diag_vals = tf.ones_like(outputs[0, :, :]) # (T_q, T_k)
tril = tf.contrib.linalg.LinearOperatorTriL(
diag_vals).to_dense() # (T_q, T_k)
masks = tf.tile(tf.expand_dims(tril, 0),
[tf.shape(outputs)[0], 1, 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(masks) * (-2**32 + 1)
outputs = tf.where(tf.equal(masks, 0), paddings,
outputs) # (h*N, T_q, T_k)
# Activation
outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
# Query Masking
query_masks = tf.sign(tf.abs(tf.reduce_sum(emb, axis=-1))) # (N, T_q)
query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
query_masks = tf.tile(tf.expand_dims(query_masks, -1),
[1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
outputs *= query_masks # broadcasting. (N, T_q, C)
# Dropouts
outputs = tf.layers.dropout(outputs,
rate=dropout_rate,
training=tf.convert_to_tensor(is_training))
# Weighted sum
outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0),
axis=2) # (N, T_q, C)
# Residual connection
outputs += queries
# Normalize
outputs = normalize(outputs) # (N, T_q, C)
return outputs
def graph(self, minibatch_size, vocab_size, input_size,
conv_size=3, conv_stride=1, conv_features=1,
weights_reg_scale=1e-6, activity_reg_scale=1e-6, embedding_reg_scale=1e-6
):
wordset_1 = tf.placeholder(
tf.int32,
shape = (minibatch_size, input_size, 1),
name = "wordset_1_indicies"
)
wordset_2 = tf.placeholder(
tf.int32,
shape = (minibatch_size, input_size, 1),
name = "wordset_2_indicies"
)
# We need to build the trainable word embedding part.
with tf.name_scope("word_vectors"):
word_vectors_init = self._initialise([vocab_size, self.embedded_word_size])
word_vectors = tf.Variable(word_vectors_init, name="word_vectors")
tf.summary.histogram("word_embedding", word_vectors)
wordset_1_vects = tf.nn.embedding_lookup(word_vectors, wordset_1)
wordset_2_vects = tf.nn.embedding_lookup(word_vectors, wordset_2)
probs = tf.placeholder(tf.float32,
shape = (minibatch_size),
name = "class_a_probability"
)
wordset_1_masks = tf.placeholder(tf.float32,
shape = (minibatch_size, input_size, 1),
name = "wordset_1_masks"
)
wordset_2_masks = tf.placeholder(tf.float32,
shape = (minibatch_size, input_size, 1),
name = "wordset_2_masks"
)
wordset_1_lengths = tf.placeholder(tf.float32,
shape = (minibatch_size),
name = "wordset_1_lengths"
)
wordset_2_lengths = tf.placeholder(tf.float32,
shape = (minibatch_size),
name = "wordset_2_lengths"
)
# Build up the convoluion. Our convolution structure will be to slide
# a window of size <conv_size>x1 across the plane. This let's us think about
# relating words near to each other.
#
# When `conv_features` is 1 we get the same number of output features
# as input words; this let's us think about these terms as describing
# word "usefulness".
with tf.name_scope("conv"):
with tf.name_scope("wordset_1"):
conv_wordset_1_init = self._initialise([conv_size, 1, self.embedded_word_size, conv_features])
conv_wordset_1_weights = tf.Variable(conv_wordset_1_init, name="weights")
conv_wordset_1_bias = tf.Variable(tf.zeros([conv_features], dtype=tf.float32), name="bias")
conv_wordset_1_y = tf.nn.conv2d(wordset_1_vects, conv_wordset_1_weights, [1, conv_stride, 1, 1], padding="SAME", name="y")
conv_wordset_1_biased = tf.nn.bias_add(conv_wordset_1_y, conv_wordset_1_bias)
wordset_1_masks_shaped = tf.expand_dims(wordset_1_masks, 3)
# Set any terms where there aren't words to zero.
conv_wordset_1_activity = tf.multiply(conv_wordset_1_biased, wordset_1_masks_shaped)
with tf.name_scope("wordset_2"):
conv_wordset_2_init = self._initialise([conv_size, 1, self.embedded_word_size, conv_features])
conv_wordset_2_weights = tf.Variable(conv_wordset_2_init, name="weights")
conv_wordset_2_bias = tf.Variable(tf.zeros([conv_features], dtype=tf.float32), name="bias")
conv_wordset_2_y = tf.nn.conv2d(wordset_2_vects, conv_wordset_2_weights, [1, conv_stride, 1, 1], padding="SAME", name="y")
conv_wordset_2_biased = tf.nn.bias_add(conv_wordset_2_y, conv_wordset_2_bias)
wordset_2_masks_shaped = tf.expand_dims(wordset_2_masks, 3)
# Set any terms where there aren't words to zero.
conv_wordset_2_activity = tf.multiply(conv_wordset_2_biased, wordset_2_masks_shaped)
# Base combination term.
mu = tf.Variable(0.0, name="mu")
with tf.name_scope("loss"):
final_means_wordset_1 = tf.div(tf.reduce_sum(conv_wordset_1_activity, reduction_indices=[1,2,3]), wordset_1_lengths)
final_means_wordset_2 = tf.div(tf.reduce_sum(conv_wordset_2_activity, reduction_indices=[1,2,3]), wordset_2_lengths)
embedding_reg_wordset_1 = tf.nn.l2_loss(wordset_1_vects)
embedding_reg_wordset_2 = tf.nn.l2_loss(wordset_2_vects)
conv_weights_reg_wordset_1 = tf.nn.l2_loss(conv_wordset_1_weights)
conv_weights_reg_wordset_2 = tf.nn.l2_loss(conv_wordset_2_weights)
activity_reg_wordset_1 = tf.reduce_mean(tf.div(tf.reduce_sum(tf.abs(conv_wordset_1_activity), reduction_indices=[1,2,3]), wordset_1_lengths))
activity_reg_wordset_2 = tf.reduce_mean(tf.div(tf.reduce_sum(tf.abs(conv_wordset_2_activity), reduction_indices=[1,2,3]), wordset_2_lengths))
# Handy debugging statements
#
# final_means_wordset_1 = tf.Print(final_means_wordset_1, [final_means_wordset_1], "FM1")
# final_means_wordset_1 = tf.Print(final_means_wordset_2, [final_means_wordset_2], "FM2")
#
# embedding_reg_wordset_1 = tf.Print(embedding_reg_wordset_1, [embedding_reg_wordset_1], "ER1")
# embedding_reg_wordset_2 = tf.Print(embedding_reg_wordset_2, [embedding_reg_wordset_2], "ER2")
# conv_weights_reg_wordset_1 = tf.Print(conv_weights_reg_wordset_1, [conv_weights_reg_wordset_1], "CR1")
# conv_weights_reg_wordset_2 = tf.Print(conv_weights_reg_wordset_2, [conv_weights_reg_wordset_2], "CR2")
# activity_reg_wordset_1 = tf.Print(activity_reg_wordset_1, [activity_reg_wordset_1], "AR1")
# activity_reg_wordset_2 = tf.Print(activity_reg_wordset_2, [activity_reg_wordset_2], "AR2")
# Total regularisation term.
regs_and_scales = [ (weights_reg_scale, conv_weights_reg_wordset_1 + conv_weights_reg_wordset_2),
(activity_reg_scale, activity_reg_wordset_1 + activity_reg_wordset_2),
(embedding_reg_scale, embedding_reg_wordset_1 + embedding_reg_wordset_2)
]
regularisation = sum(a*b for (a,b) in regs_and_scales)
# Combination term. It's a sigmoid of `mu` so that it is bounded
# between 0 and 1.
alpha = tf.sigmoid(mu, name="alpha")
tf.summary.scalar("alpha", alpha)
joint_means = alpha * final_means_wordset_1 + (1 - alpha) * final_means_wordset_2
batch_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=joint_means, labels=probs)
loss = tf.reduce_mean(batch_loss) + regularisation
tf.summary.scalar("loss", loss)
with tf.name_scope("accuracy"):
final_probs = tf.sigmoid(joint_means)
abs_diff = tf.abs(probs - final_probs)
right_enough = tf.equal(probs, tf.to_float(tf.greater(final_probs, 0.5)))
accuracy = tf.reduce_mean(tf.to_float(right_enough))
tf.summary.scalar("accuracy", accuracy)
return ModelParameters(wordset_1, wordset_2, probs,
wordset_1_masks,
wordset_2_masks,
wordset_1_lengths,
wordset_2_lengths,
conv_wordset_1_activity,
conv_wordset_2_activity,
loss,
accuracy,
conv_wordset_1_weights,
conv_wordset_2_weights,
joint_means,
final_probs,
alpha,
word_vectors)
def length(sequence):
used = tf.sign(tf.reduce_max(tf.abs(sequence), reduction_indices=2))
seq_len = tf.reduce_sum(used, reduction_indices=1) # 即每个句子的单词个数
return tf.cast(seq_len, tf.int32)
def construct_beam_search_functions(models, beam_size):
"""
Strategy:
compute the log_probs - same as with sampling
for sentences that are ended set log_prob(<eos>)=0, log_prob(not eos)=-inf
add previous cost to log_probs
run top k -> (idxs, values)
use values as new costs
divide idxs by num_classes to get state_idxs
use gather to get new states
take the remainder of idxs after num_classes to get new_predicted words
"""
# Get some parameter settings. For ensembling, some parameters are required
# to be consistent across all models but others are not. In the former
# case, we assume that consistency has already been checked. For the
# parameters that are allowed to vary across models, the first model's
# settings take precedence.
decoder = models[0].decoder
batch_size = tf.shape(decoder.init_state)[0]
embedding_size = decoder.embedding_size
translation_maxlen = decoder.translation_maxlen
target_vocab_size = decoder.target_vocab_size
high_depth = 0 if decoder.high_gru_stack == None \
else len(decoder.high_gru_stack.grus)
# Initialize loop variables
i = tf.constant(0)
init_ys = -tf.ones(dtype=tf.int32, shape=[batch_size])
init_embs = [tf.zeros(dtype=tf.float32, shape=[batch_size,embedding_size])] * len(models)
f_min = numpy.finfo(numpy.float32).min
init_cost = [0.] + [f_min]*(beam_size-1) # to force first top k are from first hypo only
init_cost = tf.constant(init_cost, dtype=tf.float32)
init_cost = tf.tile(init_cost, multiples=[batch_size/beam_size])
ys_array = tf.TensorArray(
dtype=tf.int32,
size=translation_maxlen,
clear_after_read=True,
name='y_sampled_array')
p_array = tf.TensorArray(
dtype=tf.int32,
size=translation_maxlen,
clear_after_read=True,
name='parent_idx_array')
init_base_states = [m.decoder.init_state for m in models]
init_high_states = [[m.decoder.init_state] * high_depth for m in models]
init_loop_vars = [i, init_base_states, init_high_states, init_ys, init_embs,
init_cost, ys_array, p_array]
# Prepare cost matrix for completed sentences -> Prob(EOS) = 1 and Prob(x) = 0
eos_log_probs = tf.constant(
[[0.] + ([f_min]*(target_vocab_size - 1))],
dtype=tf.float32)
eos_log_probs = tf.tile(eos_log_probs, multiples=[batch_size,1])
def cond(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
return tf.logical_and(
tf.less(i, translation_maxlen),
tf.reduce_any(tf.not_equal(prev_ys, 0)))
def body(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
# get predictions from all models and sum the log probs
sum_log_probs = None
base_states = [None] * len(models)
high_states = [None] * len(models)
for j in range(len(models)):
d = models[j].decoder
states1 = d.grustep1.forward(prev_base_states[j], prev_embs[j])
att_ctx = d.attstep.forward(states1)
base_states[j] = d.grustep2.forward(states1, att_ctx)
if d.high_gru_stack == None:
stack_output = base_states[j]
high_states[j] = []
else:
if d.high_gru_stack.context_state_size == 0:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j])
else:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j], context=att_ctx)
logits = d.predictor.get_logits(prev_embs[j], stack_output,
att_ctx, multi_step=False)
log_probs = tf.nn.log_softmax(logits) # shape (batch, vocab_size)
if sum_log_probs == None:
sum_log_probs = log_probs
else:
sum_log_probs += log_probs
# set cost of EOS to zero for completed sentences so that they are in top k
# Need to make sure only EOS is selected because a completed sentence might
# kill ongoing sentences
sum_log_probs = tf.where(tf.equal(prev_ys, 0), eos_log_probs, sum_log_probs)
all_costs = sum_log_probs + tf.expand_dims(cost, axis=1) # TODO: you might be getting NaNs here since -inf is in log_probs
all_costs = tf.reshape(all_costs,
shape=[-1, target_vocab_size * beam_size])
values, indices = tf.nn.top_k(all_costs, k=beam_size) #the sorted option is by default True, is this needed?
new_cost = tf.reshape(values, shape=[batch_size])
offsets = tf.range(
start = 0,
delta = beam_size,
limit = batch_size,
dtype=tf.int32)
offsets = tf.expand_dims(offsets, axis=1)
survivor_idxs = (indices/target_vocab_size) + offsets
new_ys = indices % target_vocab_size
survivor_idxs = tf.reshape(survivor_idxs, shape=[batch_size])
new_ys = tf.reshape(new_ys, shape=[batch_size])
new_embs = [m.decoder.y_emb_layer.forward(new_ys, factor=0) for m in models]
new_base_states = [tf.gather(s, indices=survivor_idxs) for s in base_states]
new_high_states = [[tf.gather(s, indices=survivor_idxs) for s in states] for states in high_states]
new_cost = tf.where(tf.equal(new_ys, 0), tf.abs(new_cost), new_cost)
ys_array = ys_array.write(i, value=new_ys)
p_array = p_array.write(i, value=survivor_idxs)
return i+1, new_base_states, new_high_states, new_ys, new_embs, new_cost, ys_array, p_array
final_loop_vars = tf.while_loop(
cond=cond,
body=body,
loop_vars=init_loop_vars,
back_prop=False)
i, _, _, _, _, cost, ys_array, p_array = final_loop_vars
indices = tf.range(0, i)
sampled_ys = ys_array.gather(indices)
parents = p_array.gather(indices)
cost = tf.abs(cost) #to get negative-log-likelihood
return sampled_ys, parents, cost
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 1, 1, 40], name="z")
mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask")
mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2")
z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 40], name="z_new")
istrain = tf.placeholder(tf.bool)
#z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip")
#z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv")
e = tf.placeholder(tf.float32, shape=[None, 4, 4, 552], name="e")
e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8232], name="e_p")
save_itr = 0
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# mask_ = tf.ones([FLAGS.batch_size,32,64,3])
# mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
zero = tf.zeros([FLAGS.batch_size, 1, 1, 8232])
logits, h = inference((1 - mask) * image + mask * 1.0, keep_probability, z,
0.0, istrain)
logits_e, h_e = inference((1 - mask) * image + mask * 1.0,
keep_probability, z, e, istrain)
#logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain )
#logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain )
z_pred = predictor(h, z, zero, istrain)
z_pred_e = predictor(h, z, e_p, istrain)
# z_pred_lip = predictor(h,z_lip,istrain)
# z_pred_lip_inv = predictor(h,z_lip_inv,istrain)
# logits = inference(image, keep_probability,z,istrain)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
# loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
loss_mask = tf.reduce_mean(
tf.reduce_sum(
tf.contrib.layers.flatten(tf.abs((annotation - logits) * mask)),
1))
loss_mask2 = tf.reduce_mean(
tf.reduce_sum(
tf.contrib.layers.flatten(tf.abs((annotation - logits) * mask2)),
1))
loss_ = 0.4 * loss_mask + loss_mask2
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss_)
# zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1))
zloss_ = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1))
# zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1))
# zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1))
# z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv
lip_loss_dec = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((logits - logits_e))),
1))
loss = loss_ + 0.1 * lip_loss_dec
lip_loss_pred = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))),
1))
zloss = zloss_ + 0.1 * lip_loss_pred
grads = train_z(loss_mask, z)
trainable_var = tf.trainable_variables()
trainable_z_pred_var = tf.trainable_variables(scope="predictor")
trainable_d_pred_var = tf.trainable_variables(scope="decoder")
print(trainable_z_pred_var)
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
train_pred = train_predictor(zloss, trainable_z_pred_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
saved = True
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
print(np.max(train_images))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 40))
# train_images[train_images < 0.] = -1.
# train_annotations[train_annotations < 0.] = -1.
# train_images[train_images >= 0.] = 1.0
# train_annotations[train_annotations >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
cond = random.randint(0, 10)
# saved = True
if False:
saved = False
train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(
FLAGS.batch_size)
train_images_m[train_images_m < 0.] = -1.
train_annotations_m[train_annotations_m < 0.] = -1.
train_images_m[train_images_m >= 0.] = 1.0
train_annotations_m[train_annotations_m >= 0.] = 1.0
train_images = (train_images + 1.) / 2.0 * 255.0
train_annotations = (train_annotations + 1.) / 2.0 * 255.0
train_images_m = (train_images_m + 1.) / 2.0 * 255.0
train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0
train_images_m[:, 32:, :, :] = 0
train_annotations_m[:, 32:, :, :] = 0
train_images = np.clip((train_images + train_images_m), 0.0,
255.0)
train_annotations = np.clip(
(train_annotations + train_annotations_m), 0.0, 255.0)
'''
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
'''
train_annotations_ = np.squeeze(train_annotations, axis=3)
train_images_ = train_images
train_images = train_images / 127.5 - 1.0
train_annotations = train_annotations / 127.5 - 1.0
# for itr_ in range(FLAGS.batch_size):
# utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) )
# utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) )
# train_images[:,x1:w1,y1:h1,:] = 0
# print(train_images)
r_m, r_m2 = random_mask(64)
#feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True }
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
error_dec = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 4, 4, 552))
error_dec_ = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 1, 1, 8232))
# z_l_inv = z_ + np.random.normal(0.0,0.1)
# feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True }
# z_l = z_ + np.random.normal(0.0,0.001)
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# z_l = z_ + np.random.normal(0.0,0.001)
# print("Step: %d, lip_loss:%g" % (itr,lloss))
for p in range(20):
z_ol = np.copy(z_)
# z_l = z_ol + np.random.normal(0.0,0.001)
# print("666666666666666666666666666666666666666")
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_,
e: error_dec,
mask: r_m,
mask2: r_m2,
istrain: True
}
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss))
z_loss, summ = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
z_ = np.clip(z_, -10.0, 10.0)
'''
m = interp1d([-10.0,10.0],[-1.0,1.0])
print(np.max(z_))
print(np.min(z_))
z_ol_interp = m(z_ol)
z_interp = m(z_)
_,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85})
print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
'''
# _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85})
# print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
mask: r_m,
e: error_dec,
z: z_,
mask2: r_m2,
istrain: True
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.
# valid_images[valid_images < 0.] = -1.
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
mask: r_m,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
e: error_dec,
istrain: False,
mask2: r_m2
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
if itr % 3000 == 0:
save_itr = save_itr + 3000
saver.save(sess, FLAGS.logs_dir + "model.ckpt", save_itr)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.0
# valid_images[valid_images < 0.] = -1.0
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
r_m, r_m2 = random_mask(64)
# z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 40))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 0.85,
z: z_,
istrain: False,
mask: r_m,
mask2: r_m2
}
v = 0
m__ = interp1d([-10.0, 10.0], [-1.0, 1.0])
z_ = m__(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
# print(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
# z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict)
# print("z_step: %d, Train_loss:%g" % (p,z_loss))
# z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v
# z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# print("z_____________")
# print(z__)
# print("z_")
# print(z_)
# m__ = interp1d([-10.0,10.0],[-1.0,1.0])
# z_ol = m__(z_ol)
# z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m})
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
# z_ = np.clip(z_, -1.0, 1.0)
# print(z_pred_loss)
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
istrain: False,
mask: r_m,
mask2: r_m2,
keep_probability: 0.85
})
valid_images_masked = ((1 - r_m) * valid_images + 1.) / 2.0 * 255
# valid_images = (valid_images +1.)/2.0*255
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred_ = (pred + 1.) / 2.0 * 255
# pred = pred + 1./2.0*255
pred = valid_images_masked * (1 - r_m) + pred_ * r_m
valid_annotations_ = (valid_annotations + 1.) / 2.0 * 255
# pred = np.squeeze(pred, axis=3)
print(np.max(pred))
print(valid_images.shape)
print(valid_annotations.shape)
print(pred.shape)
# for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) )
# utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) )
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) )
# print("Saved image: %d" % itr)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images_masked[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="predz_" + str(5 + itr))
utils.save_image(valid_annotations_[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
def fft_abs_for_map_fn(x):
x = (x + 1.) / 2.
x_complex = tf.complex(x, tf.zeros_like(x))[:, :, 0]
fft = tf.spectral.fft2d(x_complex)
fft_abs = tf.abs(fft)
return fft_abs
learning_rate = .01
data_path = ".././genodata_imputed_upenn_selected.txt"
Pool_data = get_data(data_path,delim= ' ')
Pool_data = reorganize(Pool_data,100)
n_steps = Pool_data.shape[1]
n_input = Pool_data.shape[2]
X, y, pred = construct_automatically(n_steps, n_hidden, n_input)
cost = tf.reduce_mean(tf.pow(y-pred,2))
#cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
error = tf.reduce_mean(tf.abs(y-pred))
init = tf.global_variables_initializer()
#This is hardcoded for my own purposes. You should change it if
#you wish to run this file
folder = "/export/mialab/users/nlewis/tf_multimodal/pretrain_models/pretrain_model_UPENN_"+time.strftime("%d_%m:%H:%M")+"_batch:_"+str(batch_size) +"_hidden_"+ str(n_hidden) + "_" + str(learning_rate)
os.makedirs(folder)
name = folder + "/model.ckpt"
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
sess.run(init)
cur_batch_size = batch_size
for epoch in range(training_iters):
for i in range(1, len(Pool_data), batch_size):
def huber_loss(x, delta=1.0):
"""Reference: https://en.wikipedia.org/wiki/Huber_loss"""
return tf.where(
tf.abs(x) < delta,
tf.square(x) * 0.5, delta * (tf.abs(x) - 0.5 * delta))
def loss(self, gpu_id):
"""
Builds models, calculates losses, saves tensorboard information.
:param gpu_id: The GPU ID to calculate losses for.
:return: Returns the generator and discriminator losses.
"""
#### general matching procedure
with tf.name_scope("losses_{}".format(gpu_id)):
before_loss = time.time()
epsilon = 1e-8
input_a, input_b, input_y_a, input_y_b, input_global_y_a, input_global_y_b, input_b_selected, input_global_y_b_selected = \
self.input_x_i[gpu_id], self.input_x_j[gpu_id], self.input_y_i[gpu_id], self.input_y_j[gpu_id], \
self.input_global_y_i[gpu_id], self.input_global_y_j[gpu_id], self.input_x_j_selected[gpu_id], \
self.input_global_y_j_selected[gpu_id]
# input_a_expand = tf.expand_dims(input_a,1)
# input_a_copy = tf.tile(input_a_expand,[1,self.support_num,1,1,1])
# current_support = tf.cond(self.z1z2_training,lambda:input_a_copy,lambda:input_b)
current_support = input_b
x_g1, z1, matching_feature, similarities1, similarities_data, z_input, z_input_2, recg_loss, KL_loss, \
reconstruction_loss, crossentropy_loss_real, crossentropy_loss_fake, accuracy_real, accuracy_fake, preds_fake = \
self.generate(input_a, current_support, input_b_selected, input_y_a, input_y_b, input_global_y_a,
input_global_y_b_selected, self.selected_classes, self.support_num, self.classes,
self.is_z2, self.is_z2_vae)
x_g2, z1, matching_feature, similarities2, similarities_data, z_input, z_input_2, recg_loss, KL_loss, \
reconstruction_loss, crossentropy_loss_real, crossentropy_loss_fake, accuracy_real, accuracy_fake, preds_fake = \
self.generate(input_a, current_support, input_b_selected, input_y_a, input_y_b, input_global_y_a,
input_global_y_b_selected, self.selected_classes, self.support_num, self.classes,
self.is_z2, self.is_z2_vae)
#### diversification loss
loss_diversification = tf.reduce_mean(tf.abs(x_g1 - x_g2))
#### mode loss
#### cycle reconstruction
feature_total = []
# similarities_onehot = tf.cast((0) * tf.ones_like(similarities[:, 0]), dtype=tf.int32)
# similarities_onehot = tf.one_hot(similarities_onehot,self.support_num)
similarities_onehot = tf.expand_dims(similarities1[:, 0], axis=-1)
similarities_index = tf.expand_dims(similarities1[:, 0], axis=-1)
#### fake image
d_real, d_fake, feature_loss, t_classification_loss, g_classification_loss, sim_loss, verification_loss, mode_loss = self.d(
input_b, x_g1, x_g2, similarities_onehot, similarities_index,
input_global_y_b[:,0], input_global_y_b_selected,
self.selected_classes, self.support_num,
self.classes, similarities1,similarities2, z1,
training=self.training_phase,
dropout_rate=self.dropout_rate)
#### distinguish interpolation coefficients
d_loss_pure, G_loss = Hinge_loss(d_real, d_fake)
if self.print:
g_classification_loss = tf.Print(g_classification_loss, [g_classification_loss], 'classification_loss',
summarize=5)
reconstruction_loss = tf.Print(reconstruction_loss, [reconstruction_loss], 'l1_reconstruction_loss',
summarize=5)
feature_loss = tf.Print(feature_loss, [feature_loss], 'matching_D_loss', summarize=5)
d_loss_pure = tf.Print(d_loss_pure, [d_loss_pure], 'D_loss', summarize=5)
G_loss = tf.Print(G_loss, [G_loss], 'G_loss', summarize=5)
##### without mask
verification_loss = tf.reduce_mean(verification_loss)
mode_loss = tf.reduce_mean(mode_loss)
sim_loss = tf.reduce_mean(sim_loss)
loss_KL = tf.reduce_mean(KL_loss)
loss_recg = tf.reduce_mean(recg_loss)
loss_reconstruction = tf.reduce_mean(reconstruction_loss)
loss_feature = tf.reduce_mean(feature_loss)
g_classification_loss = tf.reduce_mean(g_classification_loss)
t_classification_loss = tf.reduce_mean(t_classification_loss)
g_loss = G_loss * self.loss_G + g_classification_loss * self.loss_CLA + self.loss_recons_B * loss_reconstruction + \
self.loss_matching_D * loss_feature - self.loss_matching_G * verification_loss + self.loss_sim * sim_loss - self.loss_KL * mode_loss
d_loss = self.loss_D * (d_loss_pure) + self.loss_CLA * t_classification_loss
# tf.add_to_collection('fzl_losses',crossentropy_loss_real)
tf.add_to_collection('g_losses', g_loss)
tf.add_to_collection('d_losses', d_loss)
tf.add_to_collection('c_losses', t_classification_loss)
# tf.add_to_collection('recons_loss', recons_loss)
# tf.summary.scalar('G_pure_losses', G_loss_image)
# tf.summary.scalar('D_pure_losses', d_loss_image)
# tf.summary.scalar('G_pairs_losses', G_loss_sim)
# tf.summary.scalar('D_pairs_losses', d_loss_sim)
tf.summary.scalar('G_losses', G_loss)
tf.summary.scalar('D_losses', d_loss_pure)
tf.summary.scalar('sim_losses', sim_loss)
tf.summary.scalar('total_g_losses', g_loss)
tf.summary.scalar('total_d_losses', d_loss)
tf.summary.scalar('c_losses', g_classification_loss)
tf.summary.scalar('reconstruction_losses', loss_reconstruction)
tf.summary.scalar('matchingD_losses', loss_feature)
return {
"g_losses": tf.add_n(tf.get_collection('g_losses'), name='total_g_loss'),
"d_losses": tf.add_n(tf.get_collection('d_losses'), name='total_d_loss'),
"c_losses": tf.add_n(tf.get_collection('c_losses'), name='total_c_loss'),
# "fzl_losses":tf.add_n(tf.get_collection('fzl_losses'),name='total_fzl_loss'),
# "recons_losses":tf.add_n(tf.get_collection('recons_losses'),name='total_recons_loss'),
}
def preprocess(x):
#importing inside the function allows lambdas to be serialized
import tensorflow as tf
normalized = x / 255.
#return tf.image.resize_images(normalized, (224, 224)) - 0.5
#normalized = tf.image.resize_images(normalized, (224, 224))
#added = tf.reduce_sum(normalized, axis=3, keep_dims=True)
#return added / tf.reduce_max(added)
hsv = tf.image.rgb_to_hsv(normalized)
return tf.concat([normalized, hsv], 3) - 0.5
h_channel = tf.slice(hsv, [0, 0, 0, 0], [-1, -1, -1, 1])
s_channel = tf.slice(hsv, [0, 0, 0, 1], [-1, -1, -1, 1])
v_channel = tf.slice(hsv, [0, 0, 0, 2], [-1, -1, -1, 1])
false_matrix = tf.zeros_like(h_channel)
true_matrix = tf.ones_like(h_channel)
v_channel_thresholded = tf.greater(v_channel, 1. / 255. * 30.)
v_channel_with_threshold = tf.where(v_channel_thresholded, v_channel, false_matrix)
r_channel = tf.slice(normalized, [0, 0, 0, 0], [-1, -1, -1, 1])
g_channel = tf.slice(normalized, [0, 0, 0, 1], [-1, -1, -1, 1])
b_channel = tf.slice(normalized, [0, 0, 0, 2], [-1, -1, -1, 1])
#green_heavy = tf.logical_and(tf.greater(g_channel, r_channel), tf.greater(g_channel, b_channel))
#green_mask = tf.where(green_heavy, false_matrix, true_matrix)
green_heavy = tf.logical_and(tf.logical_and(tf.greater(g_channel, r_channel), tf.greater(g_channel, b_channel)), tf.less(v_channel, 75. / 255.))
green_mask = tf.where(green_heavy, true_matrix / 2., true_matrix)
v_channel_minus_green = tf.multiply(v_channel_with_threshold, green_mask)
max_value = tf.reduce_max(v_channel_minus_green)
v_channel_minus_green = v_channel_minus_green / max_value
return tf.concat([v_channel_minus_green, s_channel, h_channel, r_channel, g_channel, b_channel, v_channel, v_channel_with_threshold], 3) - 0.5
return v_channel
#return tf.concat([h_channel, s_channel, v_channel], 3)
#return s_channel
# extract colors of lines and boundary dirt
true_matrix = tf.ones_like(h_channel)
false_matrix = tf.zeros_like(h_channel)
yellow_center = 58. / 255.
red_max = 15. / 255.
brown_center = 40. / 255.
yellow_matches = tf.less(tf.abs(h_channel - yellow_center), 3. / 255.)
red_matches = tf.less(h_channel, red_max)
brown_matches = tf.less(tf.abs(h_channel - brown_center), 5. / 255.)
is_black = tf.less(v_channel, 2. / 255.)
yellow_channel = tf.where(yellow_matches, true_matrix, false_matrix) - 0.5
red_channel = tf.where(red_matches, true_matrix, false_matrix) - 0.5
brown_channel = tf.where(red_matches, true_matrix, false_matrix) - 0.5
black_channel = tf.where(is_black, true_matrix, false_matrix) - 0.5
#colors_match = tf.logical_or(is_black, tf.logical_or(brown_matches, tf.logical_or(yellow_matches, red_matches)))
#return tf.where(colors_match, true_matrix, false_matrix) - 0.5
with_extra_channels = tf.concat([normalized, s_channel, yellow_channel, red_channel, brown_channel, black_channel], 3)
return with_extra_channels
