def get_split_averages(input_tensor, input_mask, indices):
# Splits input tensor into three parts based on the indices and
# returns average of values prior to index, values at the index and
# average of values after the index.
# input_tensor: (batch_size, input_length, input_dim)
# input_mask: (batch_size, input_length)
# indices: (batch_size, 1)
# (1, input_length)
length_range = K.expand_dims(K.arange(K.shape(input_tensor)[1]), dim=0)
# (batch_size, input_length)
batched_range = K.repeat_elements(length_range, K.shape(input_tensor)[0], 0)
tiled_indices = K.repeat_elements(indices, K.shape(input_tensor)[1], 1) # (batch_size, input_length)
greater_mask = K.greater(batched_range, tiled_indices) # (batch_size, input_length)
lesser_mask = K.lesser(batched_range, tiled_indices) # (batch_size, input_length)
equal_mask = K.equal(batched_range, tiled_indices) # (batch_size, input_length)
# We also need to mask these masks using the input mask.
# (batch_size, input_length)
if input_mask is not None:
greater_mask = switch(input_mask, greater_mask, K.zeros_like(greater_mask))
lesser_mask = switch(input_mask, lesser_mask, K.zeros_like(lesser_mask))
post_sum = K.sum(switch(K.expand_dims(greater_mask), input_tensor, K.zeros_like(input_tensor)), axis=1) # (batch_size, input_dim)
pre_sum = K.sum(switch(K.expand_dims(lesser_mask), input_tensor, K.zeros_like(input_tensor)), axis=1) # (batch_size, input_dim)
values_at_indices = K.sum(switch(K.expand_dims(equal_mask), input_tensor, K.zeros_like(input_tensor)), axis=1) # (batch_size, input_dim)
post_normalizer = K.expand_dims(K.sum(greater_mask, axis=1) + K.epsilon(), dim=1) # (batch_size, 1)
pre_normalizer = K.expand_dims(K.sum(lesser_mask, axis=1) + K.epsilon(), dim=1) # (batch_size, 1)
return K.cast(pre_sum / pre_normalizer, 'float32'), values_at_indices, K.cast(post_sum / post_normalizer, 'float32')
def _build_inner_model(self, h):
dim_order = get_backend()
if len(self.input_shape) == 3:
if dim_order == 'tf':
images_number = self.input_shape[-1]
image_shape = self.input_shape[:-1]
h = Permute((3, 1, 2))(h)
h = Reshape((images_number,) + image_shape + (1,))(h)
h = Lambda(lambda t: K.repeat_elements(t, 3, 4))(h)
else:
images_number = self.input_shape[0]
image_shape = self.input_shape[1:]
h = Reshape((images_number, 1) + image_shape)(h)
h = Lambda(lambda t: K.repeat_elements(t, 3, 2))(h)
h = TimeDistributed(load_object_from_dict(self.nested_model))(h)
else:
raise NotSupportedError()
h = Flatten()(h)
h = Dense(4096, activation = 'relu')(h)
h = Dense(4096, activation = 'relu')(h)
return h
def call(self, inputs, mask=None):
if not isinstance(inputs, list) or len(inputs) <= 1:
raise TypeError('SpkLifeLongMemory must be called on a list of tensors '
'(at least 2). Got: ' + str(inputs))
# (None(batch), 1), index of speaker
target_spk_l = inputs[0]
target_spk_l = K.reshape(target_spk_l, (target_spk_l.shape[0], ))
if K.dtype(target_spk_l) != 'int32':
target_spk_l = K.cast(target_spk_l, 'int32')
# (None(batch), embed_dim)
spk_vector_l = inputs[1]
# Start to update life-long memory based on the learned speech vector
# First do normalization
spk_vector_eps = K.switch(K.equal(spk_vector_l, 0.), np.spacing(1), spk_vector_l) # avoid zero
spk_vector_eps = K.sqrt(K.sum(spk_vector_eps**2, axis=1))
spk_vector_eps = spk_vector_eps.dimshuffle((0, 'x'))
spk_vector = T.true_div(spk_vector_l, K.repeat_elements(spk_vector_eps, self.vec_dim, axis=1))
# Store speech vector into life-long memory according to the speaker identity.
life_long_mem = T.inc_subtensor(self.life_long_mem[target_spk_l, :], spk_vector)
# Normalization for memory
life_long_mem_eps = K.switch(K.equal(life_long_mem, 0.), np.spacing(1), life_long_mem) # avoid 0
life_long_mem_eps = K.sqrt(K.sum(life_long_mem_eps**2, axis=1))
life_long_mem_eps = life_long_mem_eps.dimshuffle((0, 'x'))
life_long_mem = T.true_div(life_long_mem, K.repeat_elements(life_long_mem_eps, self.vec_dim, axis=1))
# (None(batch), spk_size, embed_dim)
return life_long_mem
def step(self, x, states):
h_tild_tm1 = states[0]
B_U = states[1]
B_W = states[2]
if self.consume_less == 'cpu':
x_i = x[:, :self.output_dim]
x_f = x[:, self.output_dim: 2 * self.output_dim]
x_c = x[:, 2 * self.output_dim: 3 * self.output_dim]
x_o = x[:, 3 * self.output_dim: 4 * self.output_dim]
x_new = x[:, 4 * self.output_dim:]
else:
x_i = K.dot(x * B_W[0], self.W_i) + self.b_i
x_f = K.dot(x * B_W[1], self.W_f) + self.b_f
x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
x_new = x
# self.C_tape -> BT, t-1, k
# self.H_tape -> BT, t-1, k
# x -> BT, k
# h_tild_tm1 -> BT, k
if self.H_tape is None:
self.H_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1))
self.C_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1))
# s_t -> BT, t-1, 1
t = K.shape(self.C_tape)[1]
sum1 = K.dot(self.H_tape, self.W_h)
sum2 = K.dot(K.repeat_elements(x_new.dimshuffle((0,'x',1)),t, axis=1), self.W_x)
sum3 = K.dot(K.repeat_elements(h_tild_tm1.dimshuffle((0,'x',1)),t, axis=1), self.W_h_tilde)
tanhed_sum = K.tanh(sum1 + sum2 + sum3)
a_t = K.dot(tanhed_sum, self.v)[:,:,0]
s_t = K.softmax(a_t)
h_tilde_t = T.batched_dot(self.H_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0]
c_tilde_t = T.batched_dot(self.C_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0]
i = self.inner_activation(x_i + K.dot(h_tilde_t * B_U[0], self.U_i))
f = self.inner_activation(x_f + K.dot(h_tilde_t * B_U[1], self.U_f))
c_t = f * c_tilde_t + i * self.activation(x_c + K.dot(h_tilde_t * B_U[2], self.U_c))
o = self.inner_activation(x_o + K.dot(h_tilde_t * B_U[3], self.U_o))
h_t = o * self.activation(c_t)
# Add to Tape
self.C_tape = K.concatenate([self.C_tape, c_t.dimshuffle((0,'x',1))], axis=1)
self.H_tape = K.concatenate([self.H_tape, h_t.dimshuffle((0,'x',1))], axis=1)
return h_t, [h_tilde_t]
def call(self, X, mask=None):
if self.dim_ordering == 'th':
output = K.repeat_elements(X, self.size[0], axis=2)
output = K.repeat_elements(output, self.size[1], axis=3)
elif self.dim_ordering == 'tf':
output = K.repeat_elements(X, self.size[0], axis=1)
output = K.repeat_elements(output, self.size[1], axis=2)
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
return convauto_backend.unpool2d(self._pool2d.output, self._pool2d.input, output)
def get_output(self, train=False):
X = self.get_input(train)
if self.dim_ordering == 'th':
output = K.repeat_elements(X, self.size[0], axis=2)
output = K.repeat_elements(output, self.size[1], axis=3)
elif self.dim_ordering == 'tf':
output = K.repeat_elements(X, self.size[0], axis=1)
output = K.repeat_elements(output, self.size[1], axis=2)
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
f = T.grad(T.sum(self._pool2d_layer.get_output(train)), wrt=self._pool2d_layer.get_input(train)) * output
return f
def test_repeat_elements(self):
reps = 3
for ndims in [1, 2, 3]:
shape = np.arange(2, 2 + ndims)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
for rep_axis in range(ndims):
np_rep = np.repeat(arr, reps, axis=rep_axis)
th_z = KTH.repeat_elements(arr_th, reps, axis=rep_axis)
th_rep = KTH.eval(th_z)
tf_rep = KTF.eval(
KTF.repeat_elements(arr_tf, reps, axis=rep_axis))
assert th_rep.shape == np_rep.shape
assert tf_rep.shape == np_rep.shape
assert_allclose(np_rep, th_rep, atol=1e-05)
assert_allclose(np_rep, tf_rep, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_rep.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
shape = list(shape)
shape[rep_axis] = None
x = K.placeholder(shape=shape)
y = K.repeat_elements(x, reps, axis=rep_axis)
assert y._keras_shape == tuple(shape)
def get_initial_states(self, x):
initial_state = K.zeros_like(x) # (samples, num_steps, input_channel, h, w)
initial_state = K.sum(initial_state, [1, 2]) # (samples, h, w)
initial_state = K.expand_dims(initial_state, 1)
initial_state = K.repeat_elements(initial_state, self.nb_filter, 1)
initial_states = [initial_state for _ in range(len(self.states))]
return initial_states
def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
global add_tables
nb_filter = output_shape[0]
conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x)
conv = BatchNormalization(axis=1)(conv)
conv = Activation("relu")(conv)
conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv)
conv = BatchNormalization(axis=1)(conv)
if strides[0] >= 2:
x = AveragePooling2D(strides)(x)
if (output_shape[0] - input_shape[0]) > 0:
pad_shape = (1,
output_shape[0] - input_shape[0],
output_shape[1],
output_shape[2])
padding = K.ones(pad_shape)
padding = K.repeat_elements(padding, K.shape(x)[0], axis=0)
x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
output_shape=output_shape)(x)
_death_rate = K.variable(death_rate)
scale = K.ones_like(conv) - _death_rate
conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
output_shape=output_shape)(conv)
out = merge([conv, x], mode="sum")
out = Activation("relu")(out)
gate = K.variable(1, dtype="uint8")
add_tables += [{"death_rate": _death_rate, "gate": gate}]
return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
output_shape=output_shape)([out, x])
def call(self, x, mask=None):
if K.image_dim_ordering == "th":
_, f, r, c = self.shape
else:
_, r, c, f = self.shape
half_n = self.n // 2
squared = K.square(x)
pooled = K.pool2d(squared, (half_n, half_n), strides=(1, 1),
padding="same", pool_mode="avg")
if K.image_dim_ordering == "th":
summed = K.sum(pooled, axis=1, keepdims=True)
averaged = (self.alpha / self.n) * K.repeat_elements(summed, f, axis=1)
else:
summed = K.sum(pooled, axis=3, keepdims=True)
averaged = (self.alpha / self.n) * K.repeat_elements(summed, f, axis=3)
denom = K.pow(self.k + averaged, self.beta)
return x / denom
def call(self, x, mask=None):
# b,n,f -> b,f via b,n broadcasted
p_vectors = K.expand_dims(super(SoftAttention, self).call(x, mask), 2)
expanded_p = K.repeat_elements(p_vectors, K.shape(x)[2], axis=2)
attended = K.sum(expanded_p * x, axis=1)
if self.return_probabilities:
return [attended, p_vectors]
return attended
def _transform(target):
# Expand first dimension in any case
target = K.expand_dims(target, dim=1)
if self.axis == 2:
# Repeat target along the time dimension
target = K.repeat_elements(
target, x[0].shape[1], axis=1)
return target
def attention_control(args):
x, dense_2 = args
find_att = K.reshape(x, (15, 15, 10))
find_att = K.transpose(find_att[:, :, :])
find_att = K.mean(find_att, axis=0)
find_att = find_att / K.sum(find_att, axis=0)
find_att = K.repeat_elements(find_att, 32, axis=0)
find_att = K.reshape(find_att, (1, 32, 15, 15))
return find_att
def call(self, x, mask=None):
h, va = x[0], x[1]
wh = K.dot(h, self.Wh)
va = K.repeat_elements(va, self.time_steps, axis=1)
wv = K.dot(va, self.Wv)
m = K.concatenate([wh, wv])
if self.activation is not None:
m = self.activation(m)
alpha = K.dot(m, self.w)
alpha = K.squeeze(alpha, axis=-1)
alpha = self.softmask(alpha, mask[0])
return alpha
def build_compute_advantage(state_space,actors,critic,n_ant,n_actions,sample_num): S = Input(shape = (state_space,)) Inputs = [] for i in range(n_ant): Inputs.append(Input(shape = (n_actions,))) S_t = Lambda(lambda x: K.repeat_elements(x,sample_num,axis = 0))(S) Inputs_t = [] for i in range(n_ant): Inputs_t.append(Lambda(lambda x: K.repeat_elements(x,sample_num,axis = 0))(Inputs[i])) q = critic([S]+Inputs) Outputs = [] for i in range(n_ant): q_i = critic([S_t]+Inputs_t[0:i]+[actors[i](S_t)[2]]+Inputs_t[i+1:n_ant]) q_i = Lambda(lambda x: K.reshape(x,(-1,sample_num,1)))(q_i) Outputs.append(Lambda(lambda x: x[0] - K.mean(x[1],axis = 1))([q,q_i])) return K.function([S]+Inputs, Outputs)
def call(self, x):
mean = K.mean(x, axis=-1)
std = K.std(x, axis=-1)
if len(x.shape) == 3:
mean = K.permute_dimensions(K.repeat(mean,
x.shape.as_list()[-1]),
[0, 2, 1])
std = K.permute_dimensions(K.repeat(std,
x.shape.as_list()[-1]),
[0, 2, 1])
elif len(x.shape) == 2:
mean = K.reshape(K.repeat_elements(mean,
x.shape.as_list()[-1], 0),
(-1, x.shape.as_list()[-1]))
std = K.reshape(K.repeat_elements(mean,
x.shape.as_list()[-1], 0),
(-1, x.shape.as_list()[-1]))
return self._g * (x - mean) / (std + self._epsilon) + self._b
def call(self, X):
att_weights = self._get_attention_weights(X)
# Reshape the attention weights to match the dimensions of X
att_weights = K.reshape(att_weights, (-1, att_weights.shape[1], 1))
att_weights = K.repeat_elements(att_weights, X.shape[-1], -1)
# Multiply each input by its attention weights
weighted_input = keras.layers.Multiply()([X, att_weights])
# Sum in the direction of the time-axis.
return K.sum(weighted_input, axis=1)
def _get_loss_diag_normal(x, x_dec_param, num_samples=1):
if num_samples > 1:
x = K.repeat_elements(x, num_samples, axis=0)
x_dim = K.cast(K.shape(x)[-1], 'int32')
x_dec_param = [x_dec_param[:, :x_dim], x_dec_param[:, x_dim:]]
logPx_g_z = hyp_obj.diag_normal(x, x_dec_param)
if num_samples > 1:
r = K.reshape(logPx_g_z, (-1, num_samples))
logPx_g_z = K.mean(r, axis=1)
return logPx_g_z
def __padding_mask(self, padding_mask):
if self.compression_window_size is not None:
# Adjust only columns
padding_mask = K.expand_dims(padding_mask, axis=-1)
padding_mask = K.conv2d(padding_mask,
self.mask_conv_kernel,
strides=(1, self.compression_window_size),
padding='valid',
data_format='channels_last')
padding_mask = K.squeeze(padding_mask, axis=-1)
padding_mask = K.repeat_elements(padding_mask, self.num_heads, axis=0)
return padding_mask
def Expander(lev):
"""
Return a model that expand an input of size (lev, n_var) to (lev, lev, n_var)
"""
expand = lambda x: K.expand_dims(x, axis=-1)
repeat = lambda x: K.repeat_elements(x, lev, axis=-1)
Expand = Lambda(expand)
Repeat = Lambda(repeat)
M = Sequential()
M.add(Expand)
M.add(Repeat)
return (M)
def call(self, input_tensor, mask=None):
z_s = input_tensor[0]
z_n = input_tensor[1]
r_s = input_tensor[2]
z_s = K.l2_normalize(z_s, axis=-1)
z_n = K.l2_normalize(z_n, axis=-1)
r_s = K.l2_normalize(r_s, axis=-1)
steps = z_n.shape[1].value
pos = K.sum(z_s * r_s, axis=-1, keepdims=True)
pos = K.repeat_elements(pos, steps, axis=1)
r_s = K.expand_dims(r_s, axis=-2)
r_s = K.repeat_elements(r_s, steps, axis=1)
neg = K.sum(z_n * r_s, axis=-1)
loss = K.cast(
K.sum(K.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True),
K.floatx())
return loss
def call(self, x, mask=None):
if K.image_dim_ordering == "th":
_, f, r, c = self.shape
else:
_, r, c, f = self.shape
half_n = self.n // 2
squared = K.square(x)
pooled = K.pool2d(squared, (half_n, half_n),
strides=(1, 1),
padding="same",
pool_mode="avg")
if K.image_dim_ordering == "th":
summed = K.sum(pooled, axis=1, keepdims=True)
averaged = (self.alpha / self.n) * K.repeat_elements(
summed, f, axis=1)
else:
summed = K.sum(pooled, axis=3, keepdims=True)
averaged = (self.alpha / self.n) * K.repeat_elements(
summed, f, axis=3)
denom = K.pow(self.k + averaged, self.beta)
return x / denom
def kldivn(y_true, y_pred):
with tf.Session() as sess:
y_pred = np.float64(y_pred)
max_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(
K.max(K.max(y_pred, axis=0), axis=0)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred /= max_y_pred
sum_y_true = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(
K.sum(K.sum(y_true, axis=0), axis=0)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
sum_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(
K.sum(K.sum(y_pred, axis=0), axis=0)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
return K.sum(K.sum(
y_true * K.log((y_true / (y_pred + K.epsilon())) + K.epsilon()),
axis=-1),
axis=-1).eval()
def nss_time(y_true, y_pred):
if len(y_true.shape) == 5:
ax = 2
else:
ax = 1
maxi = K.max(K.max(y_pred, axis=ax), axis=ax)
first_rep = K.repeat_elements(K.expand_dims(maxi, axis=ax),shape_r_out, axis=ax)
max_y_pred = K.repeat_elements(K.expand_dims(first_rep, axis=ax+1), shape_c_out, axis=ax+1)
y_pred /= max_y_pred
if len(y_true.shape) == 5:
y_pred_flatten = K.reshape(y_pred, (K.shape(y_pred)[0],K.shape(y_pred)[1],K.shape(y_pred)[2]*K.shape(y_pred)[3]*K.shape(y_pred)[4])) #K.batch_flatten(y_pred)
else:
y_pred_flatten = K.batch_flatten(y_pred)
y_mean = K.mean(y_pred_flatten, axis=-1)
y_mean = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.expand_dims(y_mean)),
shape_r_out, axis=ax)), shape_c_out, axis=ax+1)
y_std = K.std(y_pred_flatten, axis=-1)
y_std = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.expand_dims(y_std)),
shape_r_out, axis=ax)), shape_c_out, axis=ax+1)
y_pred = (y_pred - y_mean) / (y_std + K.epsilon())
num = K.sum(K.sum(y_true * y_pred, axis=ax), axis=ax)
den = K.sum(K.sum(y_true, axis=ax), axis=ax) + K.epsilon()
if len(y_true.shape) == 5:
nss_out = K.mean(num/den, axis = 1)
else:
nss_out = num/den
return nss_out
def call(self, x):
a_w = self.w
a_b = self.bias
a_d = self.dilation
a_t = self.translation
a_x = x #(,input_dim)
a_x = K.repeat_elements(K.expand_dims(a_x,
axis=-2), self.wavelon_count,
-2) #(, wavelons, input_dim)
for dim in self.input_dimensions:
a_w = K.repeat_elements(K.expand_dims(a_w, 0), dim,
0) #(, output_dim, wavelon)
a_b = K.repeat_elements(K.expand_dims(a_b, 0), dim,
0) #(, output_dim, 1)
a_d = K.repeat_elements(K.expand_dims(a_d, 0), dim,
0) #(, wavelon, input_dim)
a_t = K.repeat_elements(K.expand_dims(a_t, 0), dim,
0) #(, wavelon, input_dim)
a_u = (a_x - a_t) / a_d #(, wavelons, input_dim)
psi = -a_u * K.exp(-K.square(a_u) / 2.0) #(, wavelons, input_dim)
psi = K.max(psi, axis=-1, keepdims=False) #(, wavelons)
psi = K.repeat_elements(K.expand_dims(psi, axis=-2), self.output_dim,
-2) #(, output_dim, wavelons)
xc = K.sum(a_w * psi, axis=-1, keepdims=False) + a_b #(, output_dim)
return xc
def KL(y_true, y_pred):
max_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred /= max_y_pred
sum_y_true = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.sum(K.sum(y_true, axis=2), axis=2)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
sum_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.sum(K.sum(y_pred, axis=2), axis=2)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
return 10 * K.sum(K.sum(
y_true * K.log((y_true / (y_pred + K.epsilon())) + K.epsilon()),
axis=-1),
axis=-1)
def __call__(self, p):
import numpy as np
mean_p = K.mean(p, axis=1)
(num_output, length) = K.int_shape(p)
diff1 = p[:, 1:] - p[:, :-1]
mean_diff1 = K.mean(diff1, axis=1)
diff2 = diff1[:, 1:] - diff1[:, :-1]
desired_diff2 = K.clip(diff2, -1.0 * self.m, self.m)
il1 = np.triu_indices(length - 2)
mask1 = np.ones((num_output, length - 1, length - 2))
mask1[:, il1[0], il1[1]] = 0.0
kmask1 = K.variable(value=mask1)
mat1 = kmask1 * K.repeat_elements(K.expand_dims(desired_diff2, 1),
length - 1, 1)
desired_diff1 = K.squeeze(K.squeeze(K.dot(
mat1, K.ones((1, length - 2, num_output)))[:, :, :1, :1],
axis=2),
axis=2)
desired_diff1 += K.repeat_elements(K.expand_dims(
mean_diff1 - K.mean(desired_diff1, axis=1), -1),
length - 1,
axis=1)
il2 = np.triu_indices(length - 1)
mask2 = np.ones((num_output, length, length - 1))
mask2[:, il2[0], il2[1]] = 0.0
kmask2 = K.variable(value=mask2)
mat2 = kmask2 * K.repeat_elements(K.expand_dims(desired_diff1, 1),
length, 1)
desired_p = K.squeeze(K.squeeze(K.dot(
mat2, K.ones((1, length - 1, num_output)))[:, :, :1, :1],
axis=2),
axis=2)
desired_p += K.repeat_elements(K.expand_dims(
mean_p - K.mean(desired_p, axis=1), -1),
length,
axis=1)
return desired_p
def NSS(y_true, y_pred):
max_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred /= max_y_pred
y_pred_flatten = K.batch_flatten(y_pred)
y_mean = K.mean(y_pred_flatten, axis=-1)
y_mean = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.expand_dims(y_mean)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_std = K.std(y_pred_flatten, axis=-1)
y_std = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.expand_dims(y_std)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred = (y_pred - y_mean) / (y_std + K.epsilon())
return -(K.sum(K.sum(y_true * y_pred, axis=2), axis=2) /
K.sum(K.sum(y_true, axis=2), axis=2))
def correlation_coefficient(y_true, y_pred):
max_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)),
shape_r_out, axis=-1)), shape_c_out, axis=-1)
y_pred /= max_y_pred
sum_y_true = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_true, axis=2), axis=2)),
shape_r_out, axis=-1)), shape_c_out, axis=-1)
sum_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_pred, axis=2), axis=2)),
shape_r_out, axis=-1)), shape_c_out, axis=-1)
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
N = shape_r_out * shape_c_out
sum_prod = K.sum(K.sum(y_true * y_pred, axis=2), axis=2)
sum_x = K.sum(K.sum(y_true, axis=2), axis=2)
sum_y = K.sum(K.sum(y_pred, axis=2), axis=2)
sum_x_square = K.sum(K.sum(K.square(y_true), axis=2), axis=2)
sum_y_square = K.sum(K.sum(K.square(y_pred), axis=2), axis=2)
num = sum_prod - ((sum_x * sum_y) / N)
den = K.sqrt((sum_x_square - K.square(sum_x) / N) * (sum_y_square - K.square(sum_y) / N))
# print("num is :", num)
# print("den is :", den)
return -2 * num / den # weight: -2
def cc_time(y_true, y_pred):
if len(y_true.shape) == 5:
ax = 2
else:
ax = 1
max_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=ax), axis=ax), axis=ax),
shape_r_out, axis=ax), axis=ax+1), shape_c_out, axis=ax+1)
y_pred /= max_y_pred
sum_y_true = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_true, axis=ax), axis=ax), axis=ax),
shape_r_out, axis=ax), axis=ax+1), shape_c_out, axis=ax+1)
sum_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_pred, axis=ax), axis=ax), axis=ax),
shape_r_out, axis=ax), axis=ax+1), shape_c_out, axis=ax+1)
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
N = shape_r_out * shape_c_out
sum_prod = K.sum(K.sum(y_true * y_pred, axis=ax), axis=ax)
sum_x = K.sum(K.sum(y_true, axis=ax), axis=ax)
sum_y = K.sum(K.sum(y_pred, axis=ax), axis=ax)
sum_x_square = K.sum(K.sum(K.square(y_true), axis=ax), axis=ax)
sum_y_square = K.sum(K.sum(K.square(y_pred), axis=ax), axis=ax)
num = sum_prod - ((sum_x * sum_y) / N)
den = K.sqrt((sum_x_square - K.square(sum_x) / N) * (sum_y_square - K.square(sum_y) / N))
if len(y_true.shape) == 5:
cc_out = K.mean(num / den, axis = 1)
else:
cc_out = num / den
return cc_out
def nss(y_true, y_pred):
max_y_pred = K.expand_dims(
K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.max(y_pred, axis=[2, 3, 4])),
y_pred.shape[2],
axis=2)),
y_pred.shape[3],
axis=3))
y_pred /= max_y_pred
# y_pred_flatten = K.batch_flatten(y_pred)
# max_y_true = K.expand_dims(K.repeat_elements(
# K.expand_dims(K.repeat_elements(K.expand_dims(K.max(y_true, axis=[2, 3, 4])), shape_r_out, axis=2)),
# shape_c_out, axis=3))
max_y_true = K.max(y_true, axis=[2, 3, 4])
y_bool = K.cast(K.greater(max_y_true, 0.1), 'float32')
y_mean = K.mean(y_pred, axis=[2, 3, 4])
y_mean = K.expand_dims(
K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(y_mean), y_pred.shape[2], axis=2)),
y_pred.shape[3],
axis=3))
y_std = K.std(y_pred, axis=[2, 3, 4])
y_std = K.expand_dims(
K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(y_std), y_pred.shape[2], axis=2)),
y_pred.shape[3],
axis=3))
y_pred = (y_pred - y_mean) / (y_std + K.epsilon())
return -K.sum(y_bool * ((K.sum(y_true * y_pred, axis=[2, 3, 4])) /
(K.sum(y_true, axis=[2, 3, 4]))))
def nss(y_true, y_pred):
ax = 1
if K.sum(K.sum(y_true, axis=ax), axis=ax) == 0:
return 0
max_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=ax), axis=ax), axis=ax+1),
shape_r_out, axis=ax), axis=ax+1), shape_c_out, axis=ax+1)
y_pred /= max_y_pred
y_pred_flatten = K.batch_flatten(y_pred)
y_mean = K.mean(y_pred_flatten, axis=-1)
y_mean = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.expand_dims(y_mean)),
shape_r_out, axis=ax)), shape_c_out, axis=ax+1)
y_std = K.std(y_pred_flatten, axis=-1)
y_std = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.expand_dims(y_std)),
shape_r_out, axis=ax)), shape_c_out, axis=ax+1)
y_pred = (y_pred - y_mean) / (y_std + K.epsilon())
den = K.sum(K.sum(y_true * y_pred, axis=ax), axis=ax)
nom = K.sum(K.sum(y_true, axis=ax), axis=ax) + K.epsilon()
nss_out = den/nom
return nss_out
def sim(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
y_true = tf.clip_by_value(y_true, K.epsilon(), 1 - K.epsilon())
max_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(
K.expand_dims(K.max(K.max(y_pred, axis=1), axis=1), axis=1),
y_pred.shape[1],
axis=1),
axis=2),
y_pred.shape[2],
axis=2)
y_pred /= max_y_pred
sum_y_true = K.repeat_elements(K.expand_dims(K.repeat_elements(
K.expand_dims(K.sum(K.sum(y_true, axis=1), axis=1), axis=1),
y_pred.shape[1],
axis=1),
axis=2),
y_pred.shape[2],
axis=2)
sum_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(
K.expand_dims(K.sum(K.sum(y_pred, axis=1), axis=1), axis=1),
y_pred.shape[1],
axis=1),
axis=2),
y_pred.shape[2],
axis=2)
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
return -K.sum(K.sum(K.minimum(y_true, y_pred), axis=1), axis=1)
def beta_features(x, unit_size=unit_size, max_len=max_len, beta_size=beta_size):
if mode[:8] == 'implicit' or mode == 'spatial_adaptive' or mode=='spatial_adaptive-attention':
h, vf0 = x
#h = K.l2_normalize(h, -1)
#vf0 = K.l2_normalize(vf0, -1)
vf0_ = K.repeat_elements(K.expand_dims(vf0, 1), max_len+1, 1) # [sent, 49, unit_size] or [sent, 2, unit_size]
if mode=='spatial_adaptive-attention':
h_ = K.repeat_elements(K.expand_dims(h, 2), beta_size-2, 2) # [sent, 49, unit_size]
return K.reshape(K.concatenate([h_, vf0_], 3), [-1, max_len+1, 2*(beta_size-2)*unit_size]) # [sent, 49*b*unit_size]
else:
h_ = K.repeat_elements(K.expand_dims(h, 2), beta_size-1, 2) # [sent, 49, unit_size]
return K.reshape(K.concatenate([h_, vf0_], 3), [-1, max_len+1, 2*(beta_size-1)*unit_size]) # [sent, 49*b*unit_size]
else:
h, sf, vf0 = x
#h = K.l2_normalize(h, -1)
#sf = K.l2_normalize(sf, -1)
#vf0 = K.l2_normalize(vf0, -1)
sf_ = K.expand_dims(K.repeat_elements(K.expand_dims(sf, 1), max_len+1, 1), 2) # [sent, 1, unit_size]
vf0_ = K.repeat_elements(K.expand_dims(vf0, 1), max_len+1, 1) # [sent, 49, unit_size] or [sent, 2, unit_size]
vf_sf = K.concatenate([sf_,vf0_],2) # [sent, 49+1, unit_size] or [sent, 2+1, unit_size]
h_ = K.repeat_elements(K.expand_dims(h, 2), beta_size-1, 2) # [sent, 49+1, unit_size] or or [sent, 2+1, unit_size]
return K.reshape(K.concatenate([h_, vf_sf], 3), [-1, max_len+1, 2*(beta_size-1)*unit_size]) # [sent, 49+1*b*unit_size]
def correlation_coefficient(y_true, y_pred):
max_y_pred = K.expand_dims(K.repeat_elements(
K.expand_dims(K.repeat_elements(K.expand_dims(K.max(y_pred, axis=[2, 3, 4])), y_pred.shape[2], axis=2)),
y_pred.shape[3], axis=3))
y_pred /= (max_y_pred + K.epsilon())
# max_y_true = K.expand_dims(K.repeat_elements(
# K.expand_dims(K.repeat_elements(K.expand_dims(K.max(y_true, axis=[2, 3, 4])), shape_r_out, axis=2)),
# shape_c_out, axis=3))
max_y_true = K.max(y_true, axis=[2, 3, 4])
y_bool = K.cast(K.greater(max_y_true, 0.1), 'float32')
sum_y_true = K.expand_dims(K.repeat_elements(
K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(y_true, axis=[2, 3, 4])), y_pred.shape[2], axis=2)),
y_pred.shape[3], axis=3))
sum_y_pred = K.expand_dims(K.repeat_elements(
K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(y_pred, axis=[2, 3, 4])), y_pred.shape[2], axis=2)),
y_pred.shape[3], axis=3))
y_true /= (sum_y_true + K.epsilon())
y_pred /= (sum_y_pred + K.epsilon())
N = y_pred._shape_as_list()[2] * y_pred._shape_as_list()[3]
sum_prod = K.sum(y_true * y_pred, axis=[2, 3, 4])
sum_x = K.sum(y_true, axis=[2, 3, 4])
sum_y = K.sum(y_pred, axis=[2, 3, 4])
sum_x_square = K.sum(K.square(y_true), axis=[2, 3, 4])+ K.epsilon()
sum_y_square = K.sum(K.square(y_pred), axis=[2, 3, 4])+ K.epsilon()
num = sum_prod - ((sum_x * sum_y) / N)
den = K.sqrt((sum_x_square - K.square(sum_x) / N) * (sum_y_square - K.square(sum_y) / N))
return -K.sum(y_bool*(num/(den + K.epsilon())))#
def __call__(self, q, k, v, mask=None):
d_k, d_v = self.d_k, self.d_v
n_head = self.n_head
if self.mode == 0:
qs = self.qs_layer(q) # [batch_size, len_q, n_head*d_k]
ks = self.ks_layer(k)
vs = self.vs_layer(v)
def reshape1(x):
s = tf.shape(x) # [batch_size, len_q, n_head * d_k]
x = tf.reshape(x, [s[0], s[1], n_head, s[2] // n_head])
x = tf.transpose(x, [2, 0, 1, 3])
x = tf.reshape(x, [-1, s[1], s[2] // n_head
]) # [n_head * batch_size, len_q, d_k]
return x
qs = Lambda(reshape1)(qs)
ks = Lambda(reshape1)(ks)
vs = Lambda(reshape1)(vs)
if mask is not None:
mask = Lambda(lambda x: K.repeat_elements(x, n_head, 0))(mask)
head, attn = self.attention(qs, ks, vs, mask=mask)
def reshape2(x):
s = tf.shape(x) # [n_head * batch_size, len_v, d_v]
x = tf.reshape(x, [n_head, -1, s[1], s[2]])
x = tf.transpose(x, [1, 2, 0, 3])
x = tf.reshape(x, [-1, s[1], n_head * d_v
]) # [batch_size, len_v, n_head * d_v]
return x
head = Lambda(reshape2)(head)
elif self.mode == 1:
heads = []
attns = []
for i in range(n_head):
qs = self.qs_layers[i](q)
ks = self.ks_layers[i](k)
vs = self.vs_layers[i](v)
head, attn = self.attention(qs, ks, vs, mask)
heads.append(head)
attns.append(attn)
head = Concatenate()(heads) if n_head > 1 else heads[0]
attn = Concatenate()(attns) if n_head > 1 else attns[0]
outputs = self.w_o(head)
outputs = Dropout(self.dropout)(outputs)
if not self.layer_norm: return outputs, attn
outputs = Add()([outputs, q])
return self.layer_norm(outputs), attn
def call(self, x, mask=None):
en_seq = x[0]
de_seq = x[1]
input_de_times = K.int_shape(de_seq)[-2]
if len(x) == 3:
mask = x[2]
m_en = K.cast(mask, K.floatx())
en_seq = en_seq * K.expand_dims(m_en, -1)
if len(x) == 2 and mask is not None:
# remove padding values
m_en = K.cast(mask[0], K.floatx())
en_seq = en_seq * K.expand_dims(m_en, -1)
# compute alphas
att_en = K.dot(K.reshape(en_seq, (-1, self.input_dim_en)), self.w_en)
att_en = K.reshape(att_en, shape=(-1, self.input_en_times * self.units))
att_en = K.repeat(att_en, input_de_times)
att_en = K.reshape(att_en, shape=(-1, self.input_en_times * input_de_times, self.units))
att_de = K.dot(K.reshape(de_seq, (-1, self.input_dim_de)), self.w_de)
att_de = K.reshape(att_de, shape=(-1, input_de_times, self.units))
att_de = K.repeat_elements(att_de, self.input_en_times, 1)
co_m = att_en + att_de
co_m = K.reshape(co_m, (-1, self.units))
mu = K.dot(K.tanh(co_m), self.nu)
if len(x) == 3 or (len(x) == 2 and mask is not None):
m_en = K.repeat(m_en, input_de_times)
m_en = K.reshape(m_en, shape=(-1, 1))
m_en = m_en - 1
m_en = m_en * REMOVE_FACTOR
mu = mu + m_en
mu = K.reshape(mu, shape=(-1, input_de_times, self.input_en_times))
alphas = K.softmax(mu)
en_seq = K.reshape(en_seq, shape=(-1, self.input_en_times * self.input_dim_en))
en_seq = K.repeat(en_seq, input_de_times)
en_seq = K.reshape(en_seq, shape=(-1, input_de_times, self.input_en_times, self.input_dim_en))
sum_en = K.sum(en_seq * K.expand_dims(alphas, -1), 2)
output = K.concatenate([de_seq, sum_en], -1)
if self.return_alphas:
return [output, alphas]
else:
return output
def step(self, x, states):
h, [h, c] = self.layer.step(x, states)
attention = states[4]
m = self.attn_activation(K.dot(h, self.U_a) * attention + self.b_a)
s = K.sigmoid(K.dot(m, self.U_s) + self.b_s)
if self.single_attention_param:
h = h * K.repeat_elements(s, self.layer.output_dim, axis=1)
else:
h = h * s
return h, [h, c]
def mean_logistic_loss_accuracy(y_true, y_pred):
ones = K.ones_like(y_true, 'float32')
dim = 20
y_log_loss = K.log(ones + K.exp(-y_true * y_pred))
y_score_pos = K.abs(y_true)
y_score_neg = ones - K.abs(y_true)
num_predictions = K.expand_dims(K.sum(y_score_pos, axis=1), axis=1)
num_predictions_rep = K.repeat_elements(num_predictions, rep=dim, axis=1)
predict_start = K.cast(
K.expand_dims(K.argmax(y_score_pos, axis=1), axis=1), 'float32')
predict_start_rep = K.repeat_elements(predict_start, rep=dim, axis=1)
cumsum = K.cumsum(ones, axis=1)
weights = cumsum - predict_start_rep + y_score_neg * dim
y_cumsum = K.cumsum(y_log_loss, axis=1)
y_clear = y_cumsum * y_score_pos
y_normed = (y_clear / weights) / num_predictions_rep
return K.sum(y_normed, axis=1)
def call(self, inputs, **kwargs):
image_pad = inputs[0]
regions = inputs[1]
regions = K.cast(regions, tf.float32)
regions = K.reshape(regions, (-1, 4))
regions_normalized = regions / K.cast(
K.int_shape(image_pad)[1], tf.float32)
box_ind = tf.range(K.shape(image_pad)[0])
box_ind = K.repeat_elements(box_ind, self.nbox, axis=0)
self.part_images = tf.image.crop_and_resize(image_pad,
regions_normalized,
box_ind, (self.w, self.h))
return self.part_images
def smooth_l1_loss(y_true, y_pred):
sigma = 3.0
sigma2 = sigma * sigma
isValid = K.expand_dims(y_true[:, :, 0], 2)
isValid = K.repeat_elements(isValid, 4, -1)
# flatten data
# y_pred = K.reshape(y_pred, [-1, 4])
d = K.abs(y_pred - y_true[:, :, 1:])
isLess = K.cast(d < 1 / sigma2, 'float32')
left = 0.5 * isLess * K.square(sigma * d)
right = (1 - isLess) * (d - 0.5 / sigma2)
res = K.sum((left + right) * isValid) / K.sum(isValid)
return res
def call(self, input_tensor, mask=None):
x = input_tensor[0]
y = input_tensor[1]
mask = mask[0]
y = K.transpose(K.dot(self.W, K.transpose(y)))
y = K.expand_dims(y, axis=-2)
y = K.repeat_elements(y, self.steps, axis=1)
eij = K.sum(x*y, axis=-1)
if self.bias:
b = K.repeat_elements(self.b, self.steps, axis=0)
eij += b
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
return a
def step(self, x, states):
h, params = self.layer.step(x, states)
attention = states[-1]
m = self.attn_activation(K.dot(h, self.U_a) * attention + self.b_a)
s = K.sigmoid(K.dot(m, self.U_s) + self.b_s)
if self.single_attention_param:
h = h * K.repeat_elements(s, self.layer.units, axis=1)
else:
h = h * s
return h, params
def call(self, x, mask=None):
length = self.length
shape = K.int_shape(x)
a = K.repeat_elements(x, length, axis=1)
b = K.tile(x, [1, length, 1])
x_concatenated = K.reshape(K.concatenate([a, b], axis=-1),
(-1, length, length, shape[2] * 2))
weights = self.feedforward(x_concatenated)
weights = K.reshape(weights, (-1, length))
weights = K.softmax(weights)
weighted_x_concatenated = x_concatenated * K.reshape(
weights, (-1, length, length, 1))
return K.sum(weighted_x_concatenated, axis=2)
def step(self, x, states):
h, [h, c] = super(AttentionLSTM, self).step(x, states)
attention = states[4]
m = self.attn_inner_activation(K.dot(h, self.U_a) * attention + self.b_a)
# Intuitively it makes more sense to use a sigmoid (was getting some NaN problems
# which I think might have been caused by the exponential function -> gradients blow up)
s = self.attn_activation(K.dot(m, self.U_s) + self.b_s)
if self.single_attention_param:
h = h * K.repeat_elements(s, self.output_dim, axis=1)
else:
h = h * s
return h, [h, c]
def neg_log_normal_mixture(self, true, parameters):
means = parameters[:,0*self.m:1*self.m]
sigmas = parameters[:,1*self.m:2*self.m]
pi = parameters[:,2*self.m:3*self.m]
# true_repeated = K.repeat_elements(true, self.m, axis=-1)
exponent = -0.5 * (true - means)**2 / sigmas
max_exponent = K.max(exponent, axis=-1, keepdims=True)
max_exponent_repeated = K.repeat_elements(max_exponent, self.m, axis=-1)
likelihood = pi * K.exp(exponent - max_exponent_repeated)
return K.mean(log_sum_exp(likelihood, axis=1))
def build(self, input_shape):
# The composition types are taken from Belinkov et al.'s TACL 2014 paper:
# HC: Head-Child; HPC: Head-Prep-Child; HPCT: Head-Prep-Child-Ternary.
assert self.composition_type in self.allowed_compositions, "Unknown composition type: %s" % self.composition_type
if isinstance(input_shape[0], tuple):
# This layer has multiple inputs (RelationPredictor).
input_dim = input_shape[0][-1]
input_length = input_shape[0][1]
else:
input_dim = input_shape[-1]
input_length = input_shape[1]
if self.proj_dim is None:
self.proj_dim = int(input_dim / 2)
if self.composition_type == 'HPCD':
max_num_heads = input_length - 2
# Clipping number of distance based projection matrices to 5.
num_head_projectors = min(max_num_heads, 5)
self.proj_head = self.init((num_head_projectors, input_dim, self.proj_dim))
if max_num_heads > num_head_projectors:
diff = max_num_heads - num_head_projectors
farthest_head_proj = K.expand_dims(self.proj_head[0, :, :], dim=0) # (1, input_dim, proj_dim)
# (diff, input_dim, proj_dim)
tiled_farthest_head_proj = K.repeat_elements(farthest_head_proj, diff, 0)
# (head_size, input_dim, proj_dim)
self.dist_proj_head = K.concatenate([tiled_farthest_head_proj, self.proj_head], axis=0)
else:
self.dist_proj_head = self.proj_head
else:
self.proj_head = self.init((input_dim, self.proj_dim), name='{}_proj_head'.format(self.name))
self.proj_prep = self.init((input_dim, self.proj_dim), name='{}_proj_prep'.format(self.name))
self.proj_child = self.init((input_dim, self.proj_dim), name='{}_proj_child'.format(self.name))
self.trainable_weights = [self.proj_head, self.proj_prep, self.proj_child]
self.hidden_layers = []
if self.num_hidden_layers > 0:
# This means we have to pass the composed representation through an MLP instead of directly computing
# scores.
for i in range(self.num_hidden_layers):
hidden_layer = self.init((self.proj_dim, self.proj_dim), name='%s_hidden_layer_%d' % (self.name, i))
self.hidden_layers.append(hidden_layer)
self.trainable_weights.extend(self.hidden_layers)
self.scorer = self.init((self.proj_dim, self.score_dim), name='{}_scorer'.format(self.name))
self.trainable_weights.append(self.scorer)
def attention_control(args):
# Why do we need dense_2 here!?
x, dense_2 = args
find_att = K.reshape(x, (15, 15, 10))
find_att = K.transpose(find_att[:, :, :]) # 10 x 15 x 15
find_att = K.mean(find_att, axis=0) # 15 x 15
# WTF ???
find_att = find_att / K.sum(find_att, axis=0) # 15 x 15
# TODO ??? maybe BUG: copy across channels, but he lose channel axis
find_att = K.repeat_elements(find_att, 32, axis=0)
find_att = K.reshape(find_att, (1, 32, 15, 15))
# x, dense_2 = args
# find_att = K.reshape(x, (15, 15, 10))
# find_att = K.transpose(find_att[:, :, :]) # 10 x 15 x 15
# # find average attention here across all feature maps
# mean_att = K.mean(find_att, axis=0) # 15 x 15
# mean_att = K.reshape(mean_att, (1, 15, 15))
# # copy attention mask across all feature maps
# rep_mean_att = K.repeat_elements(mean_att, 32, axis=0)
# focus = K.reshape(rep_mean_att, (1, 32, 15, 15))
# return focus
return find_att
def call(self, x, mask=None):
# input shape: (nb_samples, time (padded with zeros), input_dim)
# note that the .build() method of subclasses MUST define
# self.input_spec with a complete input shape.
input_shape = self.input_spec[0].shape
inputs = K.repeat_elements(x * 0, self.num_timesteps, 1)
initial_states = [x[:,0,:],]
constants = self.get_constants(x)
#preprocessed_input = self.preprocess_input(x)
last_output, outputs, states = K.rnn(self.step, inputs,
initial_states,
go_backwards=self.go_backwards,
mask=mask,
constants=constants,
unroll=self.unroll,
input_length=self.num_timesteps)
if self.return_sequences:
return outputs
else:
return last_output
def _interpolate(self, image, sampled_grids, output_size):
batch_size = K.shape(image)[0]
height = K.shape(image)[1]
width = K.shape(image)[2]
num_channels = K.shape(image)[3]
x = K.cast(K.flatten(sampled_grids[:, 0:1, :]), dtype='float32')
y = K.cast(K.flatten(sampled_grids[:, 1:2, :]), dtype='float32')
x = .5 * (x + 1.0) * K.cast(width, dtype='float32')
y = .5 * (y + 1.0) * K.cast(height, dtype='float32')
x0 = K.cast(x, 'int32')
x1 = x0 + 1
y0 = K.cast(y, 'int32')
y1 = y0 + 1
max_x = int(K.int_shape(image)[2] - 1)
max_y = int(K.int_shape(image)[1] - 1)
x0 = K.clip(x0, 0, max_x)
x1 = K.clip(x1, 0, max_x)
y0 = K.clip(y0, 0, max_y)
y1 = K.clip(y1, 0, max_y)
pixels_batch = K.arange(0, batch_size) * (height * width)
pixels_batch = K.expand_dims(pixels_batch, axis=-1)
flat_output_size = output_size[0] * output_size[1]
base = K.repeat_elements(pixels_batch, flat_output_size, axis=1)
base = K.flatten(base)
# base_y0 = base + (y0 * width)
base_y0 = y0 * width
base_y0 = base + base_y0
# base_y1 = base + (y1 * width)
base_y1 = y1 * width
base_y1 = base_y1 + base
indices_a = base_y0 + x0
indices_b = base_y1 + x0
indices_c = base_y0 + x1
indices_d = base_y1 + x1
flat_image = K.reshape(image, shape=(-1, num_channels))
flat_image = K.cast(flat_image, dtype='float32')
pixel_values_a = K.gather(flat_image, indices_a)
pixel_values_b = K.gather(flat_image, indices_b)
pixel_values_c = K.gather(flat_image, indices_c)
pixel_values_d = K.gather(flat_image, indices_d)
x0 = K.cast(x0, 'float32')
x1 = K.cast(x1, 'float32')
y0 = K.cast(y0, 'float32')
y1 = K.cast(y1, 'float32')
area_a = K.expand_dims(((x1 - x) * (y1 - y)), 1)
area_b = K.expand_dims(((x1 - x) * (y - y0)), 1)
area_c = K.expand_dims(((x - x0) * (y1 - y)), 1)
area_d = K.expand_dims(((x - x0) * (y - y0)), 1)
values_a = area_a * pixel_values_a
values_b = area_b * pixel_values_b
values_c = area_c * pixel_values_c
values_d = area_d * pixel_values_d
return values_a + values_b + values_c + values_d
def call(self, inputs, mask=None):
v = inputs[0]
a = inputs[1]
return K.repeat_elements(v, K.shape(a)[1], 1) + a - K.expand_dims(K.mean(a, axis=-1))
def loss(y_true, y_pred):
max_y = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)), shape_r_gt, axis=-1)), shape_c_gt, axis=-1)
return K.mean(K.square((y_pred / max_y) - y_true) / (1 - y_true + 0.1))
def repeat_1(x):
"""Wrap keras backend repeat."""
return K.repeat_elements(x, 32, 2)
def call(self, x, mask=None):
return K.repeat_elements(
K.expand_dims(x, self.axis), self.num_repeats, self.axis
)
def call(self, target_tensor, mask=None):
last_dim = K.ndim(self.p_tensor)
expanded_p = K.repeat_elements(K.expand_dims(self.p_tensor, last_dim),
K.shape(target_tensor)[last_dim],
axis=last_dim)
return K.sum(expanded_p * target_tensor, axis=last_dim-1)
