def accumulate(attend_function, inputs, input_length,
mask=None, return_probabilities=False):
'''get the running attention over a sequence.
given a 3dim tensor where the 1st dim is time (or not. whatever.), calculating the running attended sum.
in other words, at the first time step, you only have that item.
at the second time step, attend over the first two items.
at the third.. the third. so on.
this basically a mod on keras' rnn implementation
author: bcm
'''
ndim = inputs.ndim
assert ndim >= 3, 'inputs should be at least 3d'
axes = [1,0] + list(range(2, ndim))
inputs = inputs.dimshuffle(axes)
indices = list(range(input_length))
successive_outputs = []
if mask is not None:
if mask.ndim == ndim-1:
mask = K.expand_dims(mask)
assert mask.ndim == ndim
mask = mask.dimshuffle(axes)
prev_output = None
successive_outputs = []
successive_pvecs = []
uncover_mask = K.zeros_like(inputs)
uncover_indices = K.arange(input_length)
for _ in range(ndim-1):
uncover_indices = K.expand_dims(uncover_indices)
make_subset = lambda i,X: K.switch(uncover_indices <= i, X, uncover_mask)
for i in indices:
inputs_i = make_subset(i,inputs)
mask_i = make_subset(i,mask)
if mask is not None:
output = attend_function(inputs_i, mask_i) # this should not output the time dimension; it should be marginalized over.
else:
output = attend_function(inputs_i) # this should not output the time dimension; it should be marginalized over.
if return_probabilities:
output, p_vectors = output
successive_pvecs.append(p_vectors)
assert output.ndim == 2, "Your attention function is malfunctioning; the attention accumulator should return 2 dimensional tensors"
successive_outputs.append(output)
outputs = K.pack(successive_outputs)
K.squeeze(outputs, -1)
axes = [1, 0] + list(range(2, outputs.ndim))
outputs = outputs.dimshuffle(axes)
if return_probabilities:
out_pvecs = K.pack(successive_pvecs)
K.squeeze(out_pvecs, -1)
out_pvecs = out_pvecs.dimshuffle(axes)
outputs = [outputs, out_pvecs]
return outputs
def step(self, x, states): M = states[0] # (nb_samples, nb_slots, memory_size) h = states[1] # (nb_samples, memory_size) w = states[2] # (nb_samples, nb_slots) #------Memory read--------# k = self.W_k(h) # (nb_samples, memory_size) w_hat = T.batched_tensordot(M, k, axes=[(2), (1)]) # (nb_samples, nb_slots) beta = K.sigmoid(self.W_b(h)) # (nb_samples, 1) beta = K.repeat(beta, self.nb_slots) # (nb_samples, nb_slots, 1) beta = K.squeeze(beta, 2) # (nb_samples, nb_slots) w_hat = softmax(w_hat * beta) # (nb_samples, nb_slots) g = sigmoid(self.W_hg(h)) # (nb_samples, 1) g = K.repeat(g, self.nb_slots) # (nb_samples, nb_slots, 1) g = K.squeeze(g, 2) # (nb_samples, nb_slots) w = (1 - g) * w + g * w_hat # (nb_samples, nb_slots) c = T.batched_tensordot(w, M, axes=[(1), (1)]) h = tanh(self.W_ih(x) + self.W_c(c)) y = self.W_ho(h) #---------Memory write---------# v = self.W_v(h) # (nb_samples, memory_size) v = K.repeat(v, 1) e = sigmoid(self.W_he(h)) # (nb_samples, nb_slots) f = 1 - w * e # (nb_samples, nb_slots) f = K.repeat(f, self.memory_size) # (nb_samples, memory_size, nb_slots) f = K.permute_dimensions(f, (0, 2, 1)) # (nb_samples, nb_slots, memory_size) u = w # (nb_samples, nb_slots) u = K.repeat(u, 1) uv = T.batched_tensordot(u, v, axes=[(1), (1)]) M = M * f + uv return y, [M, h, w]
def squeezed_accuracy(y_true, y_pred):
class_id_true = K.argmax(K.squeeze(y_true,axis=0), axis=-1)
class_id_preds = K.argmax(K.squeeze(y_pred,axis=0), axis=-1)
# Replace class_id_preds with class_id_true for recall here
# accuracy_mask = K.cast(K.equal(class_id_preds, interesting_class_id), 'int32')
# class_acc_tensor = K.cast(K.equal(class_id_true, class_id_preds), 'int32')
class_acc = K.mean(K.equal(class_id_true, class_id_preds))
return class_acc
def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.expand_dims(x, -1)
output = K.square(K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1)))
return output
def conv_step(self, x, W, b=None, border_mode="valid", filter_shape=None, mask_type='b'):
mask = np.ones(filter_shape, dtype=_FLOATX)
in_third = self.input_dim//3
out_third = self.nb_filter//3
mask[:out_third,in_third:,0,0] = 0
mask[out_third:2*out_third,2*in_third:,0,0] = 0
W = W * mask
input_shape = self.shuffeled_dims
x = K.expand_dims(x, -1) # add a dimension of the right
conv_out = T.nnet.conv2d(x, W, subsample=self.subsample,
border_mode='half',
filter_flip=False,
input_shape=(input_shape[0],
input_shape[2],
input_shape[3],
1),
filter_shape=filter_shape)
if b:
conv_out = conv_out + K.reshape(b, (1, filter_shape[0], 1, 1))
conv_out = K.squeeze(conv_out, 3) # remove the dummy 3rd dimension
return conv_out
def step(self, x, states):
h = states[0]
# states[1] necessary?
# comes from the constants
X_static = states[-2]
# equals K.dot(static_x, self._W1) + self._b2 with X.shape=[bs, L, static_input_dim]
total_x_static_prod = states[-1]
# expand dims to add the vector which is only valid for this time step
# to total_x_prod which is valid for all time steps
hw = K.expand_dims(K.dot(h, self._W2), 1)
additive_atn = total_x_static_prod + hw
attention = K.softmax(K.dot(additive_atn, self._V), axis=1)
static_x_weighted = K.sum(attention * X_static, [1])
x = K.dot(K.concatenate([x, static_x_weighted], 1), self._W3) + self._b3
h, new_states = self.layer.cell.call(x, states[:-2])
# append attention to the states to "smuggle" it out of the RNN wrapper
attention = K.squeeze(attention, -1)
h = K.concatenate([h, attention])
return h, new_states
def call(self, X, mask=None):
# 1D -> 2D
batch = K.shape(X)[0]
width = deconv_output_length(K.shape(X)[1],
self.filter_length,
self.padding,
self.strides[2])
print("Output width: ", width)
print("Input shape: ", K.shape(X))
X = K.expand_dims(X,2)
print("Input shape after expand: ", K.shape(X))
# X = K.permute_dimensions(X, (0, 2, 3, 1))
X = K.permute_dimensions(X, (0, 2, 1, 3))
print("Input shape after permute: ", K.shape(X))
deconv_shape = tf.pack([batch, 1, width, self.nb_filter])
print("Deconv shape: ", deconv_shape)
conv_out = tf.nn.conv2d_transpose(X, self.W, strides=self.strides,
padding=self.padding.upper(),
output_shape=deconv_shape)
output = conv_out + K.reshape(self.b, (1, 1, 1, self.W_shape[2]))
print("Output shape: ", K.shape(output))
# output = K.permute_dimensions(output, (0, 3, 1, 2))
output = K.permute_dimensions(output, (0, 2, 1, 3))
print("Output shape after permute: ", K.shape(output))
# 2D -> 1D
output = K.squeeze(output,2)
print("Output shape after squeeze: ", K.shape(output))
return output
def call(self, x):
assert(K.backend() == 'tensorflow')
temp = K.permute_dimensions(x, (0, 2, 1))
for i in range(0, self.attention_depth):
temp = K.sigmoid(K.dot(temp, self.Ws[i]) + self.bs[i])
temp = K.permute_dimensions(temp, (0, 2, 1))
estimated_weight = K.squeeze(K.dot(temp, K.expand_dims(self.Wf, -1)), -1)
biased_weight = estimated_weight + self.bias
non_linear_weight = K.tanh(biased_weight)
# For each hidded state calculate how much should it contribute
# to the context vector. This is the main part of attention.
# In order to convert weights to "probabilities" use a sigmoid
# based function: exp(x) / sum(exp(xi)).
prob = K.exp(non_linear_weight)
# Compute the total sum for each batch.
total_sum = K.sum(prob, axis=1, keepdims=True)
prob /= K.cast(total_sum, K.floatx())
# Enable this if you want access to internal probabilities.
# Should only be used for testing that Attention works as expected.
# return prob
# Multiply each hidden value by the corresponding probability.
prob = K.expand_dims(prob, -1)
new_hidden_values = x * prob
return K.sum(new_hidden_values, axis=1)
def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.reshape(x, (-1, self.input_length))
x = K.expand_dims(x, 1)
x = K.expand_dims(x, -1)
if self.real_filts is not None:
conv_out_r = K.conv2d(x, self.W_r, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
else:
conv_out_r = x
if self.complex_filts is not None:
conv_out_c1 = K.conv2d(x, self.W_c1, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
conv_out_c2 = K.conv2d(x, self.W_c2, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
conv_out_c = K.sqrt(K.square(conv_out_c1) + K.square(conv_out_c2) + K.epsilon())
output = K.concatenate((conv_out_r, conv_out_c), axis=1)
else:
output = conv_out_r
output_shape = self.get_output_shape_for((None, self.input_length, self.input_dim))
output = K.squeeze(output, 3) # remove the dummy 3rd dimension
output = K.permute_dimensions(output, (2, 1, 0))
output = K.reshape(output, (-1, output_shape[1], output.shape[1]*output.shape[2]))
return output
def Conv1DTranspose(input_tensor, filters, kernel_size, strides=2, padding='same'):
x = Lambda(lambda x: K.expand_dims(x, axis=2))(input_tensor)
#x = Reshape( ( -1, 1 ) )( input_tensor )
#x = Permute( ( 3, 1, 2 ) )( x )
x = Conv2DTranspose(filters=filters, kernel_size=(kernel_size, 1), strides=(1, 1), padding=padding, kernel_initializer = 'he_normal')(x)
x = Lambda(lambda x: K.squeeze(x, axis=2))(x)
x = Permute( ( 2, 1 ) )( x )
return x
def reverse(x):
if K.ndim(x) == 2:
x = K.expand_dims(x, -1)
rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
rev = K.squeeze(rev, -1)
else:
rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
return K.permute_dimensions(rev, (1, 0, 2))
def _triplet_loss(self, labels: Tensor, pairwise_dist: Tensor) -> Tensor :
y_true = K.squeeze(labels, axis=1)
"""Triplet loss function"""
if self.hard_triplets:
triplet_loss = self._batch_hard_triplet_loss(y_true, pairwise_dist)
else:
triplet_loss = self._batch_all_triplet_loss(y_true, pairwise_dist)
return triplet_loss
def call(self, x, mask=None):
energy = K.squeeze(self.layer(x), 2)
p_matrix = softmax(energy)
if mask is not None:
mask = self.squash_mask(mask)
p_matrix = make_safe(p_matrix * mask) # remove unwanted items
p_matrix = p_matrix / K.sum(p_matrix, axis=-1, keepdims=True) # renormalize
return make_safe(p_matrix)
def step(self, x, states):
ytm, stm = states
# repeat the hidden state to the length of the sequence
_stm = K.repeat(stm, self.timesteps)
# now multiplty the weight matrix with the repeated hidden state
_Wxstm = K.dot(_stm, self.W_a)
# calculate the attention probabilities
# this relates how much other timesteps contributed to this one.
et = K.dot(activations.tanh(_Wxstm + self._uxpb),
K.expand_dims(self.V_a))
at = K.exp(et)
at_sum = K.sum(at, axis=1)
at_sum_repeated = K.repeat(at_sum, self.timesteps)
at /= at_sum_repeated # vector of size (batchsize, timesteps, 1)
# calculate the context vector
context = K.squeeze(K.batch_dot(at, self.x_seq, axes=1), axis=1)
# ~~~> calculate new hidden state
# first calculate the "r" gate:
rt = activations.sigmoid(
K.dot(ytm, self.W_r)
+ K.dot(stm, self.U_r)
+ K.dot(context, self.C_r)
+ self.b_r)
# now calculate the "z" gate
zt = activations.sigmoid(
K.dot(ytm, self.W_z)
+ K.dot(stm, self.U_z)
+ K.dot(context, self.C_z)
+ self.b_z)
# calculate the proposal hidden state:
s_tp = activations.tanh(
K.dot(ytm, self.W_p)
+ K.dot((rt * stm), self.U_p)
+ K.dot(context, self.C_p)
+ self.b_p)
# new hidden state:
st = (1-zt)*stm + zt * s_tp
yt = activations.softmax(
K.dot(ytm, self.W_o)
+ K.dot(stm, self.U_o)
+ K.dot(context, self.C_o)
+ self.b_o)
if self.return_probabilities:
return at, [yt, st]
else:
return yt, [yt, st]
def call(self, x, mask=None):
# x[0]: (batch_size, input_length, input_dim)
# x[1]: (batch_size, 1) indices of prepositions
# Optional: x[2]: (batch_size, input_length - 2)
assert isinstance(x, list) or isinstance(x, tuple)
encoded_sentence = x[0]
prep_indices = K.squeeze(x[1], axis=-1) #(batch_size,)
batch_indices = K.arange(K.shape(encoded_sentence)[0]) # (batch_size,)
if self.with_attachment_probs:
# We're essentially doing K.argmax(x[2]) here, but argmax is not differentiable!
head_probs = x[2]
head_probs_padding = K.zeros_like(x[2])[:, :2] # (batch_size, 2)
# (batch_size, input_length)
padded_head_probs = K.concatenate([head_probs, head_probs_padding])
# (batch_size, 1)
max_head_probs = K.expand_dims(K.max(padded_head_probs, axis=1))
# (batch_size, input_length, 1)
max_head_prob_indices = K.expand_dims(K.equal(padded_head_probs, max_head_probs))
# (batch_size, input_length, input_dim)
masked_head_encoding = K.switch(max_head_prob_indices, encoded_sentence, K.zeros_like(encoded_sentence))
# (batch_size, input_dim)
head_encoding = K.sum(masked_head_encoding, axis=1)
else:
head_indices = prep_indices - 1 # (batch_size,)
head_encoding = encoded_sentence[batch_indices, head_indices, :] # (batch_size, input_dim)
prep_encoding = encoded_sentence[batch_indices, prep_indices, :] # (batch_size, input_dim)
child_encoding = encoded_sentence[batch_indices, prep_indices+1, :] # (batch_size, input_dim)
'''
prep_indices = x[1]
sentence_mask = mask[0]
if sentence_mask is not None:
if K.ndim(sentence_mask) > 2:
# This means this layer came after a Bidirectional layer. Keras has this bug which
# concatenates input masks instead of output masks.
# TODO: Fix Bidirectional instead.
sentence_mask = K.any(sentence_mask, axis=(-2, -1))
head_encoding, prep_encoding, child_encoding = self.get_split_averages(encoded_sentence, sentence_mask,
prep_indices)
'''
head_projection = K.dot(head_encoding, self.proj_head) # (batch_size, proj_dim)
prep_projection = K.dot(prep_encoding, self.proj_prep) # (batch_size, proj_dim)
child_projection = K.dot(child_encoding, self.proj_child) # (batch_size, proj_dim)
#(batch_size, proj_dim)
if self.composition_type == 'HPCT':
composed_projection = K.tanh(head_projection + prep_projection + child_projection)
elif self.composition_type == 'HPC':
prep_child_projection = K.tanh(prep_projection + child_projection) # (batch_size, proj_dim)
composed_projection = K.tanh(head_projection + prep_child_projection)
else:
# Composition type in HC
composed_projection = K.tanh(head_projection + child_projection)
for hidden_layer in self.hidden_layers:
composed_projection = K.tanh(K.dot(composed_projection, hidden_layer)) # (batch_size, proj_dim)
# (batch_size, num_classes)
class_scores = K.dot(composed_projection, self.scorer)
label_probabilities = K.softmax(class_scores)
return label_probabilities
def create_cnn_network(config):
#also look at GaussianNoise and GaussianDropout
# epsilon = 1e-06
# mode = 0
# momentum=0.9
# weights=weights
# model.add(BatchNormalization(epsilon=epsilon, mode=mode, momentum=momentum, weights=weights))
#TODO: add dropout and batchnorm to encoders and decoders
graph = Graph()
input_shape,depth_axis,steps_axis = calc_input_shape(config)
graph.add_input(name='input', input_shape=input_shape)
if config['dim_ordering'] == 'tf':
steps_axis = 2
elif config['dim_ordering'] == 'th':
steps_axis = 3
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
#make num input steps odd
graph.add_node(TDSlice1D(input_shape[steps_axis]-1,0), name='sliced_input', input='input')
#TODO: add input distorter here as data augmentation
googlenet = TDGoogleNet1D(graph, 'sliced_input', input_shape, config)
googlenet_output = googlenet.result
output_shape = googlenet.output_shape
#make sure we output a flattened layer
assert output_shape[steps_axis] == 1
squeezed_shape = (output_shape[0], output_shape[1], output_shape[depth_axis])
squeezed_output = 'squeezed_output'
graph.add_node(Lambda(lambda x: K.squeeze(x,steps_axis),output_shape = squeezed_shape),
name=squeezed_output,input=googlenet_output)
recurrent_input = squeezed_output
recurrent_input_shape = squeezed_shape
#could add a fully connected layer to change dimension instead of using the output dimension
#graph.add(TimeDistributedDense(input_dim=squeezed_shape[2], output_dim=config['num_hidden_dimensions']),
# name="encoder2rnn_fc",input=squeezed_output)
#recurrent_input = "encoder2rnn_fc"
#recurrent_input_shape = squeezed_shape
#recurrent_input_shape[2] = config['num_hidden_dimensions']
if use_lstm:
cell = LSTM
else:
cell = GRU
for cur_unit in xrange(num_recurrent_units):
rnn_cell_name = "rnn_cell_%d"%cur_unit
graph.add_node(cell(input_dim=recurrent_input_shape[2], output_dim=recurrent_input_shape[2], return_sequences=True),input=recurrent_input,name=name)
recurrent_input = rnn_cell_name
graph.add_output(name="output", input=rnn_cell_name)
graph.compile(optimizer='rmsprop', loss={'output':'mse'})
#TODO:add classifier here
return graph
def _step(tensor):
model_output = model(tensor)
output = K.squeeze(model_output, axis=1)
for layer in model.layers:
if issubclass(type(layer), Recurrent):
# TODO this is not being entered!
layer.initial_state = layer.final_states
feedback_result = feedback_function(output)
# if type(output) not in (list, tuple):
# output = [output]
return output + [feedback_result]
def call(self, X, mask=None):
# 1D -> 2D
X = K.expand_dims(X,2)
X = K.permute_dimensions(X, (0, 2, 3, 1))
conv_out = tf.nn.conv2d_transpose(X, self.W, strides=self.strides,
padding=self.padding.upper(),
output_shape=self.deconv_shape)
output = conv_out + K.reshape(self.b, (1, 1, 1, self.W_shape[2]))
output = K.permute_dimensions(output, (0, 3, 1, 2))
# 2D -> 1D
output = K.squeeze(output,2)
return output
def dot_product(x, kernel):
"""
Wrapper for dot product operation, in order to be compatible with both
Theano and Tensorflow
Args:
x (): input
kernel (): weights
Returns:
"""
if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
else:
return K.dot(x, kernel)
def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.expand_dims(x, -1)
conv_out = K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1))
conv_out_s = conv_out[:,:,:self.nb_simple]
conv_out_c = K.square(conv_out[:,:,self.nb_simple:])
output = K.concatenate((conv_out_s, conv_out_c), axis=-1)
return output
def step(self, x, states):
h_tm1, c_tm1, y_tm1, B, U, H = states
s = K.dot(c_tm1, self.W_h) + self.b_h
s = K.repeat(s, self.input_length)
energy = time_distributed_dense(s + H, self.W_a, self.b_a)
energy = K.squeeze(energy, 2)
alpha = K.softmax(energy)
alpha = K.repeat(alpha, self.input_dim)
alpha = K.permute_dimensions(alpha, (0, 2, 1))
weighted_H = H * alpha
v = K.sum(weighted_H, axis=1)
y, new_states = super(AttentionDecoder, self).step(v, states[:-1])
return y, new_states
def create_score_model(self) -> Model:
cr = self.model.inputs
if self.triplet_mode:
emb_c = self.model.get_layer("sentence_embedding").get_output_at(0)
emb_r = self.model.get_layer("sentence_embedding").get_output_at(1)
dist_score = Lambda(lambda x: self._euclidian_dist(x), name="score_model")
score = dist_score([emb_c, emb_r])
else:
score = self.model.get_layer("score_model").output
score = Lambda(lambda x: 1. - K.squeeze(x, -1))(score)
score = Lambda(lambda x: 1. - x)(score)
model = Model(cr, score)
return model
def get_output(self, train=False):
X = train
X = K.expand_dims(X, -1) # add a dimension of the right
X = K.permute_dimensions(X, (0, 2, 1, 3))
conv_out = K.conv2d(X, self.W, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
output = self.activation(output)
output = K.squeeze(output, 3) # remove the dummy 3rd dimension
output = K.permute_dimensions(output, (0, 2, 1))
return output
def ctc_cost(y_true, y_pred):
'''
CTC cost:
a theano wrapper for warp-ctc
Arguments:
y_true : label
y_pred : acts
'''
from theano_ctc import ctc_cost as warp_ctc_cost
# convert (batch size, timestep, target) to (timestep, batch size, target)
acts = K.permute_dimensions(y_pred, (1, 0, 2))
labels = K.cast(K.squeeze(y_true, axis=2), 'int32')
return warp_ctc_cost(acts, labels)
def call(self, x, mask=None):
# x: (batch_size, input_length, input_dim) where input_length = head_size + 2
head_encoding = x[:, :-2, :] # (batch_size, head_size, input_dim)
prep_encoding = x[:, -2, :] # (batch_size, input_dim)
child_encoding = x[:, -1, :] # (batch_size, input_dim)
if self.composition_type == 'HPCD':
# TODO: The following line may not work with TF.
# (batch_size, head_size, input_dim, 1) * (1, head_size, input_dim, proj_dim)
head_proj_prod = K.expand_dims(head_encoding) * K.expand_dims(self.dist_proj_head, dim=0)
head_projection = K.sum(head_proj_prod, axis=2) # (batch_size, head_size, proj_dim)
else:
head_projection = K.dot(head_encoding, self.proj_head) # (batch_size, head_size, proj_dim)
prep_projection = K.expand_dims(K.dot(prep_encoding, self.proj_prep), dim=1) # (batch_size, 1, proj_dim)
child_projection = K.expand_dims(K.dot(child_encoding, self.proj_child), dim=1) # (batch_size, 1, proj_dim)
#(batch_size, head_size, proj_dim)
if self.composition_type == 'HPCT':
composed_projection = K.tanh(head_projection + prep_projection + child_projection)
elif self.composition_type == 'HPC' or self.composition_type == "HPCD":
prep_child_projection = K.tanh(prep_projection + child_projection) # (batch_size, 1, proj_dim)
composed_projection = K.tanh(head_projection + prep_child_projection)
else:
# Composition type in HC
composed_projection = K.tanh(head_projection + child_projection)
for hidden_layer in self.hidden_layers:
composed_projection = K.tanh(K.dot(composed_projection, hidden_layer)) # (batch_size, head_size, proj_dim)
# (batch_size, head_size)
head_word_scores = K.squeeze(K.dot(composed_projection, self.scorer), axis=-1)
if mask is None:
attachment_probabilities = K.softmax(head_word_scores) # (batch_size, head_size)
else:
if K.ndim(mask) > 2:
# This means this layer came after a Bidirectional layer. Keras has this bug which
# concatenates input masks instead of output masks.
# TODO: Fix Bidirectional instead.
mask = K.any(mask, axis=(-2, -1))
# We need to do a masked softmax.
exp_scores = K.exp(head_word_scores) # (batch_size, head_size)
head_mask = mask[:, :-2] # (batch_size, head_size)
# (batch_size, head_size)
masked_exp_scores = switch(head_mask, exp_scores, K.zeros_like(head_encoding[:, :, 0]))
# (batch_size, 1). Adding epsilon to avoid divison by 0. But epsilon is float64.
exp_sum = K.cast(K.expand_dims(K.sum(masked_exp_scores, axis=1) + K.epsilon()), 'float32')
attachment_probabilities = masked_exp_scores / exp_sum # (batch_size, head_size)
return attachment_probabilities
def call(self, x, mask=None):
# size of x :[batch_size, sel_len, attention_dim]
# size of u :[batch_size, attention_dim]
# uit = tanh(xW+b)
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
ait = K.squeeze(ait, -1)
ait = K.exp(ait)
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
ait *= K.cast(mask, K.floatx())
ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())
ait = K.expand_dims(ait)
weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
return output
def conv_step_hidden(self, x, W, border_mode="valid", filters=None, filter_shape=None):
input_shape = self.shuffeled_dims
if filters == None:
filters = self.nb_filter
x = K.expand_dims(x, -1) # add a dimension of the right
conv_out = T.nnet.conv2d(x, W, subsample=(1, 1),
border_mode='valid',
filter_flip=False,
input_shape=(input_shape[0],
filters,
input_shape[3],
1),
filter_shape=filter_shape)
conv_out = K.squeeze(conv_out, 3) # remove the dummy 3rd dimension
return conv_out
def step(self, x_input, states):
input_shape = self.input_spec[0].shape
en_seq = states[-1]
_, [h, c] = super(PointerLSTM, self).step(x_input, states[:-1])
# vt*tanh(W1*e+W2*d)
dec_seq = K.repeat(h, input_shape[1])
#dec_seq = K.repeat(h, 2)
print ('dec_seq')
print (dec_seq)
Eij = time_distributed_dense(en_seq, self.W1, output_dim=1)
Dij = time_distributed_dense(dec_seq, self.W2, output_dim=1)
U = self.vt * tanh(Eij + Dij)
print ('U')
print (U)
U = K.squeeze(U, 2)
print ('U squeezed')
print (U)
# make probability tensor
pointer = softmax(U)
return pointer, [h, c]
def special_cube_conv(in_tensor, filter_size):
"""
Takes in a None (samples) x 54 x ? (filters) tensor.
It embedds it into 5 x 5 grid, and does a 3D convolution
using only the nodes in the orginal embedding.
To speed things up, it actually does the folowing:
- pads the end with a zero (in the last dimension):
None (samples) x 55 x ? (filters) (neighbors)
- align neighbors to get an output of dim:
None (samples) x 54 x 27 x ? (filters) (neighbors)
- 2d convolution with filter (1, 27) and no padding to get an output of dim:
None (samples) x 54 x filter_size
- reshape to remove last dimension:
None (samples) x filter_size x 54
"""
print("in ", in_tensor.shape)
# pad (output dim: None x 55 x ?)
padded = Lambda(lambda x: K.temporal_padding(x, (0, 1)))(in_tensor) # just pad end
print("padded", padded.shape)
# align neighbors (output dim: None x 54 x 27 x ?)
#aligned = K.gather(padded, neighbors)
#aligned = padded[ neighbors[np.newaxis].astype(np.int32), :]
aligned = Lambda(lambda x: tf.gather(x, neighbors, axis=1))(padded)
print("align ", aligned.shape)
# 2D convolution in one axis (output dim: None x 54 x 1 x filter_size)
conv = Conv2D(filter_size, kernel_size=(1, 27),
strides=(1, 1),
padding='valid',
data_format="channels_last",
kernel_regularizer=l2(0.001),
bias_regularizer=l2(0.001))(aligned)
print("conv ", conv.shape)
# reshape (output dim: None x 54 x filter_size)
out_tensor = Lambda(lambda x: K.squeeze(x, axis=2))(conv)
return out_tensor
def call(self, x):
assert(K.backend() == 'tensorflow')
# The model is described by the following equations:
# estimated_weight_i = dot_product(hidden_state_i, W).
# biased_weight = estimated_weight + bias
# non_linear_weight = tanh(biased_weight)
estimated_weight = K.squeeze(K.dot(x, K.expand_dims(self.W, -1)), -1)
biased_weight = estimated_weight + self.bias
non_linear_weight = K.tanh(biased_weight)
# For each hidded state calculate how much should it contribute
# to the context vector. This is the main part of attention.
# In order to convert weights to "probabilities" use a sigmoid
# based function: exp(x) / sum(exp(xi)).
prob = K.exp(non_linear_weight)
# Compute the total sum for each batch.
total_sum = K.sum(prob, axis=1, keepdims=True)
prob /= K.cast(total_sum, K.floatx())
# Multiply each hidden value by the corresponding probability.
prob = K.expand_dims(prob, -1)
new_hidden_values = x * prob
return K.sum(new_hidden_values, axis=1)
def squeeze(axis):
return Lambda(lambda x: K.squeeze(x, axis))
def call(self, inputs, mask=None):
'''
:param inputs: a list of tensor of length not larger than 2, or a memory tensor of size BxTXD1.
If a list, the first entry is memory, and the second one is query tensor of size BxD2 if any
:param mask: the masking entry will be directly discarded
:return: a tensor of size BxD1, weighted summing along the sequence dimension
'''
query = None
if isinstance(inputs, list):
memory = inputs[0]
if len(inputs) > 1:
query = inputs[1]
elif len(inputs) > 2:
raise ValueError('inputs length should not be larger than 2')
if isinstance(mask, list):
mask = mask[0]
else:
memory = inputs
input_shape = K.int_shape(memory)
if len(input_shape) >3:
input_length = input_shape[1]
memory = K.reshape(memory, (-1,) + input_shape[2:])
if mask is not None:
mask = K.reshape(mask, (-1,) + input_shape[2:-1])
if query is not None:
raise ValueError('query can be not supported')
last = memory[:,-1,:]
memory = memory[:,:-1,:]
if query is None:
query = last
else:
query = K.concatenate([query, last], axis=-1)
if self.method is None:
if len(input_shape) > 3:
output_shape = K.int_shape(last)
return K.reshape(last, (-1, input_shape[1], output_shape[-1]))
else:
return last
elif self.method == 'cba':
hidden = K.dot(memory, self.Wh) + K.expand_dims(K.dot(query, self.Wq), 1)
hidden = K.tanh(hidden)
s = K.squeeze(K.dot(hidden, self.v), -1)
elif self.method == 'ga':
s = K.sum(K.expand_dims(K.dot(query, self.Wq), 1) * memory, axis=-1)
else:
s = K.squeeze(K.dot(memory, self.v), -1)
s = K.softmax(s)
if mask is not None:
mask = mask[:,:-1]
s *= K.cast(mask, dtype='float32')
sum_by_time = K.sum(s, axis=-1, keepdims=True)
s = s / (sum_by_time + K.epsilon())
#return [K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1), s]
result = K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1)
if len(input_shape)>3:
output_shape = K.int_shape(result)
return K.reshape(result, (-1, input_shape[1], output_shape[-1]))
else:
return result
def squeeze(tensor):
return K.squeeze(tensor, axis=-1)
def compute_similarity(self, repeated_context_vectors, repeated_query_vectors):
element_wise_multiply = repeated_context_vectors * repeated_query_vectors
concatenated_tensor = K.concatenate(
[repeated_context_vectors, repeated_query_vectors, element_wise_multiply], axis=-1)
dot_product = K.squeeze(K.dot(concatenated_tensor, self.kernel), axis=-1)
return linear(dot_product + self.bias)
def center_caculate(a,b):
stage1_center = a + K.squeeze(b, axis=1)
end_points['stage1_center'] = stage1_center
return stage1_center
def get_intersection(y_true, y_pred):
""" aka TP, assuming binary images that were removed the background """
return K.sum(K.sum(K.squeeze(y_true * y_pred, axis=3), axis=2), axis=1)
def call(self, q_embed, mask=None):
q_emb_exp = K.squeeze(q_embed, axis=self.dim)
# q_emb_exp = K.squeeze(q_embed)
show_layer_info('squeeze 1', q_emb_exp)
return q_emb_exp
def __init__(
self,
model,
bounds,
channel_axis=3,
preprocessing=(0, 1),
predicts='probabilities'):
super(KerasModel, self).__init__(bounds=bounds,
channel_axis=channel_axis,
preprocessing=preprocessing)
from keras import backend as K
import keras
self._task = 'cls'
if predicts == 'probs':
predicts = 'probabilities'
assert predicts in ['probabilities', 'logits']
images_input = model.input
label_input = K.placeholder(shape=(1,))
predictions = model.output
shape = K.int_shape(predictions)
_, num_classes = shape
assert num_classes is not None
self._num_classes = num_classes
if predicts == 'probabilities':
if K.backend() == 'tensorflow':
predictions, = predictions.op.inputs
loss = K.sparse_categorical_crossentropy(
label_input, predictions, from_logits=True)
elif predicts == 'logits':
loss = K.sparse_categorical_crossentropy(
label_input, predictions, from_logits=True)
# sparse_categorical_crossentropy returns 1-dim tensor,
# gradients wants 0-dim tensor
loss = K.squeeze(loss, axis=0)
grads = K.gradients(loss, images_input)
grad_loss_output = K.placeholder(shape=(num_classes, 1))
external_loss = K.dot(predictions, grad_loss_output)
# remove batch dimension of predictions
external_loss = K.squeeze(external_loss, axis=0)
# remove singleton dimension of grad_loss_output
external_loss = K.squeeze(external_loss, axis=0)
grads_loss_input = K.gradients(external_loss, images_input)
if K.backend() == 'tensorflow':
# tensorflow backend returns a list with the gradient
# as the only element, even if loss is a single scalar tensor
assert isinstance(grads, list)
assert len(grads) == 1
grad = grads[0]
assert isinstance(grads_loss_input, list)
assert len(grads_loss_input) == 1
grad_loss_input = grads_loss_input[0]
self._loss_fn = K.function(
[images_input, label_input], [loss])
self._batch_pred_fn = K.function(
[images_input], [predictions])
self._pred_grad_fn = K.function(
[images_input, label_input], [predictions, grad])
self._bw_grad_fn = K.function(
[grad_loss_output, images_input], [grad_loss_input])
def call(self, x):
first_token = K.squeeze(x[:, 0:1, :], axis=1)
return K.tanh(K.dot(first_token, self.pool))
def toGrey(self, img):
# returns a 2D grey tensor of the given 4D tensor
img = K.squeeze(img, 0)
img = img[:, :, 0] * 0.2126 + img[:, :, 1] * 0.7152 + img[:, :,
2] * 0.0722
return img
def get_model():
input_shape = (None, None, 2)
y0 = Input(shape=input_shape)
y1_pitch = Conv2D(32, (5, 5),
padding='same',
activation='relu',
name='pitch_layer1')(y0)
y1a_pitch = BatchNormalization()(y1_pitch)
y2_pitch = Conv2D(32, (5, 5),
padding='same',
activation='relu',
name='pitch_layer2')(y1a_pitch)
y2a_pitch = BatchNormalization()(y2_pitch)
y3_pitch = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='smoothy2')(y2a_pitch)
y3a_pitch = BatchNormalization()(y3_pitch)
y4_pitch = Conv2D(8, (70, 3),
padding='same',
activation='relu',
name='distribute')(y3a_pitch)
y4a_pitch = BatchNormalization()(y4_pitch)
y_multif0 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
name='multif0_presqueeze')(y4a_pitch)
multif0 = Lambda(lambda x: K.squeeze(x, axis=3), name='multif0')(y_multif0)
y_mask = Multiply(name='mask')([y_multif0, y0])
y1_timbre = Conv2D(512, (2, 3),
padding='same',
activation='relu',
name='timbre_layer1')(y_mask)
y1a_timbre = BatchNormalization()(y1_timbre)
y_concat = Concatenate(name='timbre_and_pitch')([y_multif0, y1a_timbre])
ya_concat = BatchNormalization()(y_concat)
y_mel_feat = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='melody_filters')(ya_concat) #32
ya_mel_feat = BatchNormalization()(y_mel_feat)
y_mel_feat2 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='melody_filters2')(ya_mel_feat) #32
ya_mel_feat2 = BatchNormalization()(y_mel_feat2)
y_mel_feat3 = Conv2D(8, (240, 1),
padding='same',
activation='relu',
name='melody_filters3')(ya_mel_feat2) # 8
ya_mel_feat3 = BatchNormalization()(y_mel_feat3)
y_mel_feat4 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='melody_filters4')(ya_mel_feat3) # 16
ya_mel_feat4 = BatchNormalization()(y_mel_feat4)
y_mel_feat5 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='melody_filters5')(ya_mel_feat4) #16
ya_mel_feat5 = BatchNormalization()(y_mel_feat5)
y_bass_feat = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='bass_filters')(ya_concat) #32
ya_bass_feat = BatchNormalization()(y_bass_feat)
y_bass_feat2 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='bass_filters2')(ya_bass_feat) #32
ya_bass_feat2 = BatchNormalization()(y_bass_feat2)
y_bass_feat3 = Conv2D(8, (240, 1),
padding='same',
activation='relu',
name='bass_filters3')(ya_bass_feat2) #8
ya_bass_feat3 = BatchNormalization()(y_bass_feat3)
y_bass_feat4 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='bass_filters4')(ya_bass_feat3) #16
ya_bass_feat4 = BatchNormalization()(y_bass_feat4)
y_bass_feat5 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='bass_filters5')(ya_bass_feat4) #16
ya_bass_feat5 = BatchNormalization()(y_bass_feat5)
y_vocal_feat = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='vocal_filters')(ya_concat) #32
ya_vocal_feat = BatchNormalization()(y_vocal_feat)
y_vocal_feat2 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='vocal_filters2')(ya_vocal_feat) #32
ya_vocal_feat2 = BatchNormalization()(y_vocal_feat2)
y_vocal_feat3 = Conv2D(8, (240, 1),
padding='same',
activation='relu',
name='vocal_filters3')(ya_vocal_feat2) #8
ya_vocal_feat3 = BatchNormalization()(y_vocal_feat3)
y_vocal_feat4 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='vocal_filters4')(ya_vocal_feat3) # 16
ya_vocal_feat4 = BatchNormalization()(y_vocal_feat4)
y_vocal_feat5 = Conv2D(16, (7, 7),
padding='same',
activation='relu',
name='vocal_filters5')(ya_vocal_feat4) #16
ya_vocal_feat5 = BatchNormalization()(y_vocal_feat5)
y_melody = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
name='melody_presqueeze')(ya_mel_feat5)
melody = Lambda(lambda x: K.squeeze(x, axis=3), name='melody')(y_melody)
y_bass = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
name='bass_presqueeze')(ya_bass_feat5)
bass = Lambda(lambda x: K.squeeze(x, axis=3), name='bass')(y_bass)
y_vocal = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
name='vocal_presqueeze')(ya_vocal_feat5)
vocal = Lambda(lambda x: K.squeeze(x, axis=3), name='vocal')(y_vocal)
model = Model(inputs=y0, outputs=[multif0, melody, bass, vocal])
model.summary(line_length=120)
return model
def _sample(self, inputs, mask = None):
substrings_by_size = []
binaries_by_size = []
masks_by_size = []
inputs_padded = self._pad(inputs)
if not mask is None:
mask_padded = self._pad(mask)
batchsize = K.shape(inputs_padded)[0]
timesteps = K.shape(inputs_padded)[1]
int_shape = K.int_shape(inputs)
def sample_length(_):
return K.random_uniform(shape = (1,), minval=self.low, maxval=self.high, dtype = "int32")
all_lengths = K.arange(self.low, self.high)
samples_left = self.samples - (self.high - self.low)
all_lengths = K.concatenate([all_lengths,
K.squeeze(K.map_fn(sample_length, K.arange(0, samples_left), dtype = "int32"), axis = -1)], axis = 0)
for length in range(self.low, self.high):
times_sampled = K.sum(K.cast(K.equal(all_lengths, length), "int32"))
def sample_start(_):
return K.random_uniform(shape=(1,), minval=0, maxval=timesteps - length, dtype = "int32")
starts = K.squeeze(K.map_fn(sample_start, K.arange(times_sampled), dtype = "int32"), axis = -1)
def sample_substring(start):
return inputs_padded[:,start:start+length]
substrings = K.map_fn(sample_substring, starts, dtype = K.dtype(inputs))
# (samples, batchsize, length, ...)
substrings = K.permute_dimensions(substrings, [1,0] + list(range(2, K.ndim(substrings))))
substrings._keras_shape = (None, None, length) + int_shape[2:]
substrings._uses_learning_phase = inputs._uses_learning_phase
substrings_by_size.append(substrings)
def sample_binary(start):
before = K.zeros((batchsize, start))
ones = K.ones((batchsize, length))
after = K.zeros((batchsize, timesteps - start - length))
return K.concatenate([before, ones, after], axis = 1)
binaries = K.map_fn(sample_binary, starts, dtype = K.floatx())
# (samples, batchsize, timesteps)
binaries = K.permute_dimensions(binaries, [1,0,2])
# (batchsize, samples, timesteps)
binaries_by_size.append(binaries)
if not mask is None:
def sample_submask(start):
return mask_padded[:,start:start+length]
masks = K.map_fn(sample_submask, starts, dtype = K.dtype(mask))
# (samples, batchsize, length)
masks = K.permute_dimensions(masks, [1,0,2])
masks_by_size.append(masks)
return substrings_by_size, binaries_by_size, masks_by_size
def compute_similarity(self, tensor_1, tensor_2):
combined_tensors = self._combine_tensors(tensor_1, tensor_2)
dot_product = K.squeeze(K.dot(combined_tensors, self.weight_vector),
axis=-1)
return self.activation(dot_product + self.bias)
def step(self, inputs, states):
h_tm1 = states[0]
c_tm1 = states[1]
dp_mask = states[2]
rec_dp_mask = states[3]
x_input = states[4]
# alignment model
h_att = K.repeat(h_tm1, self.timestep_dim)
att = _time_distributed_dense(x_input,
self.attention_weights,
self.attention_bias,
output_dim=K.int_shape(
self.attention_weights)[1])
attention_ = self.attention_activation(
K.dot(h_att, self.attention_recurrent_weights) + att) # energy
attention_ = K.squeeze(K.dot(attention_,
self.attention_recurrent_bias),
2) # energy
alpha = K.exp(attention_)
if dp_mask is not None:
alpha *= dp_mask[0]
alpha /= K.sum(alpha, axis=1, keepdims=True)
alpha_r = K.repeat(alpha, self.input_dim)
alpha_r = K.permute_dimensions(alpha_r, (0, 2, 1))
# make context vector (soft attention after Bahdanau et al.)
z_hat = x_input * alpha_r
context_sequence = z_hat
z_hat = K.sum(z_hat, axis=1)
if self.implementation == 2:
z = K.dot(inputs * dp_mask[0], self.kernel)
z += K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel)
z += K.dot(z_hat, self.attention_kernel)
if self.use_bias:
z = K.bias_add(z, self.bias)
z0 = z[:, :self.units]
z1 = z[:, self.units:2 * self.units]
z2 = z[:, 2 * self.units:3 * self.units]
z3 = z[:, 3 * self.units:]
i = self.recurrent_activation(z0)
f = self.recurrent_activation(z1)
c = f * c_tm1 + i * self.activation(z2)
o = self.recurrent_activation(z3)
else:
if self.implementation == 0:
x_i = inputs[:, :self.units]
x_f = inputs[:, self.units:2 * self.units]
x_c = inputs[:, 2 * self.units:3 * self.units]
x_o = inputs[:, 3 * self.units:]
elif self.implementation == 1:
x_i = K.dot(inputs * dp_mask[0], self.kernel_i) + self.bias_i
x_f = K.dot(inputs * dp_mask[1], self.kernel_f) + self.bias_f
x_c = K.dot(inputs * dp_mask[2], self.kernel_c) + self.bias_c
x_o = K.dot(inputs * dp_mask[3], self.kernel_o) + self.bias_o
else:
raise ValueError('Unknown `implementation` mode.')
i = self.recurrent_activation(
x_i + K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel_i) +
K.dot(z_hat, self.attention_i))
f = self.recurrent_activation(
x_f + K.dot(h_tm1 * rec_dp_mask[1], self.recurrent_kernel_f) +
K.dot(z_hat, self.attention_f))
c = f * c_tm1 + i * self.activation(
x_c + K.dot(h_tm1 * rec_dp_mask[2], self.recurrent_kernel_c) +
K.dot(z_hat, self.attention_c))
o = self.recurrent_activation(
x_o + K.dot(h_tm1 * rec_dp_mask[3], self.recurrent_kernel_o) +
K.dot(z_hat, self.attention_o))
h = o * self.activation(c)
if 0 < self.dropout + self.recurrent_dropout:
h._uses_learning_phase = True
if self.return_attention:
return context_sequence, [h, c]
else:
return h, [h, c]
plt.plot(cnn_data[num, :], linewidth=4, color='black')
plt.show()
# 获取窗口数据
X, timestep = slide_window(X, window, step)
X = np.expand_dims(X, axis=-1)
# 构建模型
input0 = Input(shape=(X.shape[1], X.shape[2], 1), name='input0')
conv2d = Conv2D(filters=1,
kernel_size=[1, 4],
strides=1,
padding='valid',
use_bias=0,
name='conv2d',
trainable=True)
tensor = conv2d(input0)
tensor = Lambda(lambda x: K.squeeze(x, axis=-1), name='squeeze_0')(tensor)
tensor = Lambda(lambda x: tf.nn.top_k(x, x.shape[2])[0],
name='sequential_pooling_drop')(tensor)
tensor = Lambda(lambda x: K.reverse(x, axes=2),
name='sequential_pooling_asend')(tensor)
tensor = LSTM(units=40, name='lstm')(tensor)
tensor = Dense(20, activation='tanh', name='FC0')(tensor)
prediction = Dense(1, activation='sigmoid', name='output',
use_bias='False')(tensor)
model = Model(inputs=input0, outputs=prediction)
# 加载训练好的模型权重
model.load_weights(r'.\model\CNN_SP_LSTM\\' + str(signal_len) +
's-train_2-fold.h5py')
layer_name = 'sequential_pooling_asend'
intermediate_layer_model = Model(input=model.input,
def get_model(self, num_classes, activation='sigmoid'):
max_len = opt.max_len
voca_size = opt.unigram_hash_size + 1
with tf.device('/gpu:0'):
embd = Embedding(voca_size, opt.embd_size, name='word_embd')
word = Input((max_len, ), name="word")
word_embed = embd(word)
img = Input((2048, ), name="image")
permute_layer = Lambda(
lambda x: K.permute_dimensions(x, (0, 2, 1)))
dot_product = Lambda(
lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))
softmax = Activation('softmax')
squeeze = Lambda(lambda x: K.squeeze(x, axis=2))
num_attns = 4
char_attns = []
for i in range(num_attns):
e_char = Dense(1, activation='softmax')(word_embed)
alpha_char = permute_layer(e_char)
char_attn = dot_product([word_embed, alpha_char])
char_attn = squeeze(char_attn)
char_attns.append(char_attn)
# concatenate context vector
char_context = concatenate(char_attns, axis=1)
h_char = Dense(1)(char_context)
h_img = Dense(1)(img)
h_w = concatenate([h_char, h_img], axis=1)
h_w = softmax(h_w)
# allocate weights
h_w_charb = Lambda(lambda x: x[:, 0], output_shape=(1, ))(h_w)
h_w_img = Lambda(lambda x: x[:, 1], output_shape=(1, ))(h_w)
h_w_charb = Reshape((1, ))(h_w_charb)
h_w_img = Reshape((1, ))(h_w_img)
h_w_charb = RepeatVector(128 * num_attns)(h_w_charb)
h_w_img = RepeatVector(2048)(h_w_img)
h_w_char = Reshape((128 * num_attns, ))(h_w_charb)
h_w_img = Reshape((2048, ))(h_w_img)
h1_char = multiply([char_context, h_w_char])
h1_img = multiply([img, h_w_img])
concat_size = 128 * num_attns + 2048
h1 = concatenate([h1_char, h1_img], axis=1)
h1 = Reshape((concat_size, ))(h1)
h1 = Dropout(0.5)(h1)
res1 = self.residual(h1, h1, concat_size)
res2 = self.residual(h1, res1, concat_size)
outputs = Dense(num_classes, activation=activation)(res2)
model = Model(inputs=[word, img], outputs=outputs)
optm = keras.optimizers.Nadam(opt.lr)
model.compile(loss='binary_crossentropy',
optimizer=optm,
metrics=[top1_acc])
model.summary(print_fn=lambda x: self.logger.info(x))
return model
def content_layer(passage_modeling):
relu_encoding = Dense(Params.embedding_dim, activation="relu")(passage_modeling) #Dense(100, name="content_activation", activation="relu")(passage_context)
indice_probability = Dense(1, activation="sigmoid")(relu_encoding) #Dense(100, name="content_indice", activation="sigmoid")(relu_encoding)
indice_probability = K.squeeze(indice_probability, axis=-1)
return indice_probability
def _backward_step(gamma_t, states):
y_tm1 = K.squeeze(states[0], 0)
y_t = batch_gather(gamma_t, y_tm1)
return y_t, [K.expand_dims(y_t, 0)]
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
#logits = K.log(y_pred / (1. - y_pred))
logits = y_pred #Jiaxin
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
return loss
def sparse_loss(self, y_true, y_pred):
# Linear Chain Conditional Random Field loss function with sparse tag sequences.
y_true = K.cast(y_true, 'int32')
y_true = K.squeeze(y_true, 2)
mask = self._fetch_mask()
return sparse_chain_crf_loss(y_true, y_pred, self.U, self.b_start, self.b_end, mask)
def squeeze(inputs, squeeze_aixs):
return K.squeeze(inputs,squeeze_aixs)
def construt_self_attention_policy_output(self):
tiled_pose = self.get_tiled_input(self.pose_input)
concat_input = Concatenate(axis=2)([
self.key_config_input, self.goal_flag_input, self.collision_input,
tiled_pose
])
dim_input = concat_input.shape[2]._value
# The query matrix
query = self.create_conv_layers(concat_input,
dim_input,
use_pooling=False,
use_flatten=False)
query = Conv2D(filters=1,
kernel_size=(1, 1),
strides=(1, 1),
activation='linear',
kernel_initializer=self.kernel_initializer,
bias_initializer=self.bias_initializer)(query)
# The key matrix
"""
key = self.create_conv_layers(concat_input, dim_input, use_pooling=False, use_flatten=False)
key = Conv2D(filters=256,
kernel_size=(1, 1),
strides=(1, 1),
activation='linear',
kernel_initializer=self.kernel_initializer,
bias_initializer=self.bias_initializer)(key)
def compute_W(x):
qvals = x[0]
kvals = x[1]
qvals = tf.transpose(qvals, perm=[0, 1, 3, 2])
dotted = tf.keras.backend.batch_dot(kvals, qvals) /tf.sqrt(tf.dtypes.cast(qvals.shape[2]._value,tf.float32))
dotted = tf.squeeze(dotted, axis=-1)
dotted = tf.squeeze(dotted, axis=-1)
return K.softmax(dotted, axis=-1)
W = Lambda(compute_W, name='softmax')([query, key])
"""
def compute_W(x):
x = K.squeeze(x, axis=-1)
x = K.squeeze(x, axis=-1)
return K.softmax(x * 100, axis=-1)
W = Lambda(compute_W, name='softmax')(query)
self.w_model = Model(inputs=[
self.goal_flag_input, self.key_config_input, self.collision_input,
self.pose_input
],
outputs=W,
name='w_model')
# The value matrix
value = self.create_conv_layers(concat_input,
dim_input,
use_pooling=False,
use_flatten=False)
value = Conv2D(
filters=4,
kernel_size=(1, 1),
strides=(1, 1),
activation='linear',
kernel_initializer=self.kernel_initializer,
bias_initializer=self.bias_initializer,
)(value)
# value = self.key_config_input
"""
value = Lambda(lambda x: K.squeeze(x, axis=-1))(self.key_config_input)
"""
value = Lambda(lambda x: K.squeeze(x, axis=2),
name='key_config_transformation')(value)
self.value_model = Model(inputs=[
self.goal_flag_input, self.key_config_input, self.collision_input,
self.pose_input
],
outputs=value,
name='value_model')
output = Lambda(lambda x: K.batch_dot(x[0], x[1]))([W, value])
return output
def call(self, inputs, **kwargs):
if inputs.get_shape().ndims == 5:
assert inputs.get_shape(
)[-2].value == 1, 'Error: Must have num_capsules = 1 going into Length'
inputs = K.squeeze(inputs, axis=-2)
return K.expand_dims(tf.norm(inputs, axis=-1), axis=-1)
def compute_W(x):
x = K.squeeze(x, axis=-1)
x = K.squeeze(x, axis=-1)
return K.softmax(x * 100, axis=-1)
def pad_backend(inputs, in_channels, out_channels):
pad_dim = (out_channels - in_channels) // 2
inputs = K.expand_dims(inputs, -1)
inputs = K.spatial_3d_padding(inputs, ((0, 0), (0, 0), (pad_dim, pad_dim)),
'channels_last')
return K.squeeze(inputs, -1)
def main():
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
input1 = Input(shape=(2, 3), name="input1") # Context
input2 = Input(shape=(4, 3), name="input2") # Query
# sim = Similarity()([input1, input2])
sim = Input(shape=(2, 4), name="sim")
# Context-to-query attention.
# Softmax on each line.
col_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(sim)
# Product between sofmax prob and each query vector.
UT = Lambda(
lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
output_shape=lambda x: x[0][:2] + x[1][2:])([col_softmax, input2])
# Query-to-context attention.
# Max per line then softmax.
line_softmax = Lambda(lambda x: K.max(x, axis=-1),
output_shape=lambda x: (x[0], x[1]))(sim)
line_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(line_softmax)
# Make @line_softmax a matrix with 1 row.
line_softmax = Lambda(lambda x: K.expand_dims(x, axis=1),
output_shape=lambda x: (x[0], 1, x[1]))(line_softmax)
# Matrix multiplication.
HT = Lambda(
lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
output_shape=lambda x: x[0][:2] + x[1][2:])([line_softmax, input1])
# Remove one extra row.
HT = Lambda(lambda x: K.squeeze(x, axis=1),
output_shape=lambda x: (x[0], x[2]))(HT)
HT = RepeatVector(2)(HT) # CONTEXT_LEN TIMES
G = Concatenate(axis=-1)(
[input1, UT,
Multiply()([input1, UT]),
Multiply()([input1, HT])])
model = Model(inputs=[input1, input2, sim], outputs=[G])
model.compile(loss=categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
model.summary()
'''
y = model.predict({
'input1': np.array([
[[1, 2, 3], [4, 5, 6]],
[[-0.5, -1, -1.5], [-2, -2.5, -3]]
]),
'input2': np.array([
[[7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]],
[[-13, -14, -15], [-7, -8, -9], [-10, -11, 0], [-16, -17, -18]]
])
}, batch_size=2)
'''
y = model.predict(
{
'input1':
np.array([[[1, 2, 3], [4, 5, 6]], [[-0.5, -1, -1.5],
[-2, -2.5, -3]]]),
'input2':
np.array([[[7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]],
[[-13, -14, -15], [-7, -8, -9], [-10, -11, 0],
[-16, -17, -18]]]),
'sim':
np.array([[[-2, -2.1, -1.9, -2], [3, 2.8, 3.2, 0]],
[[1, 1.5, 0.5, 0.9], [6, 5, 7, 5.5]]])
},
batch_size=2)
import time
time.sleep(60 * 50)
print(y)
'''
def create_network(input_shape, prior, L1=0, L2=0, dropout=0, KERNEL_NUM = 64):
inp = Input(shape=input_shape)
depth = Lambda(lambda x: x[:,:,:,2], name='extract_depth')(inp)
depth = expand_dims(axis=-1)(depth)
color = Lambda(lambda x: x[:,:,:,:2], name='extract_color')(inp)
half_kernel = KERNEL_NUM//2
# Depth Path
x = Conv2D(half_kernel, 7, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(depth)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
# First Residual Stage
x = residual_stage(x, half_kernel, iter=2, L1=L1, L2=L2, dropout=dropout)
# Downsampling
x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
# Second Residual Stage
x = residual_stage(x, 2*half_kernel, iter=3, L1=L1, L2=L2, dropout=dropout)
# Downsampling
x = Conv2D(2*half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
depth = x
# Color Path
x = Conv2D(half_kernel, 7, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(color)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
# First Residual Stage
x = residual_stage(x, half_kernel, iter=2, L1=L1, L2=L2, dropout=dropout)
# Downsampling
x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
# Second Residual Stage
x = residual_stage(x, 2*half_kernel, iter=3, L1=L1, L2=L2, dropout=dropout)
# Downsampling
x = Conv2D(2*half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dropout(dropout)(x)
color = x
# Combine
combined = concatenate([color, depth], axis=-1)
# Third Residual Stage
features_at_depth = []
for i in range(8):
x = residual_stage(combined, half_kernel, iter=5, L1=L1, L2=L2, dropout=dropout)
x = bilinear_resize((8,8))(x)
x = expand_dims(axis=1)(x)
features_at_depth.append(x)
x = concatenate(features_at_depth, axis=1)
# Trainable residual 3D blocks
x = residual_block(x, bottleneck_kernels=2*KERNEL_NUM,
out_kernels=8*KERNEL_NUM,
kernel_size=3,
identity=False,
conv=Conv3D,
L1=L1,
L2=L2)
x = Dropout(dropout)(x)
for i in range(2): # Originally 2
x = residual_block(x, bottleneck_kernels=2*KERNEL_NUM,
out_kernels=8*KERNEL_NUM,
kernel_size=3,
identity=True,
conv=Conv3D,
L1=L1,
L2=L2)
x = Dropout(dropout)(x)
x = Conv3D(1, 3, padding='same', activation='tanh')(x)
x = Lambda(lambda x: K.squeeze(x, axis=-1))(x)
def add_prior(x, prior=prior):
prior = K.constant(K.cast_to_floatx(prior))
prior = expand_dims(axis=0)(prior)
prior = Lambda( lambda prior: K.tile( prior, (K.shape(x)[0], 1, 1, 1) ))(prior)
return x + prior
x = Lambda( lambda x: add_prior(x) )(x)
return Model(inp, x)
def squeeze_layer(arr, axis=1):
return K.squeeze(arr, axis)
def get_alls(y_true, y_pred):
""" aka TP + TP + FP + FN, assuming binary images that were removed the background """
return K.sum(K.sum(K.squeeze(y_true + y_pred, axis=3), axis=2), axis=1)
def _traspose(x):
x = K.squeeze(x, axis=-2)
x = K.expand_dims(x, axis=-1)
return x