def step(self, x, states):
h = states[0]
# states[1] necessary?
# comes from the constants
X_static = states[-2]
# equals K.dot(static_x, self._W1) + self._b2 with X.shape=[bs, L, static_input_dim]
total_x_static_prod = states[-1]
# expand dims to add the vector which is only valid for this time step
# to total_x_prod which is valid for all time steps
hw = K.expand_dims(K.dot(h, self._W2), 1)
additive_atn = total_x_static_prod + hw
attention = K.softmax(K.dot(additive_atn, self._V), axis=1)
static_x_weighted = K.sum(attention * X_static, [1])
x = K.dot(K.concatenate([x, static_x_weighted], 1), self._W3) + self._b3
h, new_states = self.layer.cell.call(x, states[:-2])
# append attention to the states to "smuggle" it out of the RNN wrapper
attention = K.squeeze(attention, -1)
h = K.concatenate([h, attention])
return h, new_states
def inner_loss(y_true, y_pred):
# Workaround until https://github.com/plaidml/plaidml/pull/284 is accepted
if K.backend() == "plaidml.keras.backend":
y_true = K.reshape(y_true, y_pred.shape.dims)
n_true = K.concatenate([y_true[:, :, :, i:i+1] * mask for i in range(3)], axis=-1)
n_pred = K.concatenate([y_pred[:, :, :, i:i+1] * mask for i in range(3)], axis=-1)
return loss_func(n_true, n_pred)
def correct_boxes(box_xy, box_wh, input_shape, image_shape):
'''Get corrected boxes'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_maxes = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], # y_min
box_mins[..., 1:2], # x_min
box_maxes[..., 0:1], # y_max
box_maxes[..., 1:2] # x_max
])
# Scale boxes back to original image shape.
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def batched_dot(input, W_list, b_list, sizes_list):
X = input
W = K.concatenate(W_list, axis=-1)
b = K.concatenate(b_list, axis=-1)
Y = K.dot(X, W) + b
sl = [0, ] + list(np.cumsum(sizes_list))
return [Y[:, sl[i]:sl[i+1]] for i in range(len(sizes_list))]
def test_sparse_concat(self):
x_d = np.array([0, 7, 2, 3], dtype=np.float32)
x_r = np.array([0, 2, 2, 3], dtype=np.int64)
x_c = np.array([4, 3, 2, 3], dtype=np.int64)
x_sparse_1 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5))
x_d = np.array([0, 7, 2, 3], dtype=np.float32)
x_r = np.array([0, 2, 2, 3], dtype=np.int64)
x_c = np.array([4, 3, 2, 3], dtype=np.int64)
x_sparse_2 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5))
x_dense_1 = x_sparse_1.toarray()
x_dense_2 = x_sparse_2.toarray()
backends = [KTF]
if KTH.th_sparse_module:
# Theano has some dependency issues for sparse
backends.append(KTH)
for K in backends:
k_s = K.concatenate([K.variable(x_sparse_1), K.variable(x_sparse_2)])
assert K.is_sparse(k_s)
k_s_d = K.eval(k_s)
k_d = K.eval(K.concatenate([K.variable(x_dense_1), K.variable(x_dense_2)]))
assert k_s_d.shape == k_d.shape
assert_allclose(k_s_d, k_d, atol=1e-05)
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(T, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
yolo_outputs = args[:3]
y_true = args[3:]
anchor_mask = [[6,7,8], [3,4,5], [0,1,2]]
input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(3)]
loss = 0
m = K.shape(yolo_outputs[0])[0]
for l in range(3):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
pred_xy, pred_wh, pred_confidence, pred_class_probs = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]], num_classes, input_shape)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet box loss.
xy_delta = (y_true[l][..., :2]-pred_xy)*grid_shapes[l][::-1]
wh_delta = K.log(y_true[l][..., 2:4]) - K.log(pred_wh)
# Avoid log(0)=-inf.
wh_delta = K.switch(object_mask, wh_delta, K.zeros_like(wh_delta))
box_delta = K.concatenate([xy_delta, wh_delta], axis=-1)
box_delta_scale = 2 - y_true[l][...,2:3]*y_true[l][...,3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box)))
return b+1, ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
box_loss = object_mask * K.square(box_delta*box_delta_scale)
confidence_loss = object_mask * K.square(1-pred_confidence) + \
(1-object_mask) * K.square(0-pred_confidence) * ignore_mask
class_loss = object_mask * K.square(true_class_probs-pred_class_probs)
loss += K.sum(box_loss) + K.sum(confidence_loss) + K.sum(class_loss)
return loss / K.cast(m, K.dtype(loss))
def call(self, argument, mask=None):
"""Execute this layer on input tensors.
Parameters
----------
argument: list
List of two tensors (X, Xp). X should be of shape (n_test, n_feat) and
Xp should be of shape (n_support, n_feat) where n_test is the size of
the test set, n_support that of the support set, and n_feat is the number
of per-atom features.
Returns
-------
list
Returns two tensors of same shape as input. Namely the output shape will
be [(n_test, n_feat), (n_support, n_feat)]
"""
x, xp = argument
# Get initializations
p = self.p_init
q = self.q_init
# Rename support
z = xp
states = self.support_states_init
x_states = self.test_states_init
for d in range(self.max_depth):
# Process support xp using attention
e = cos(z + q, xp)
a = K.softmax(e)
# Get linear combination of support set
r = K.dot(a, xp)
# Not sure if it helps to place the update here or later yet. Will
# decide
# z = r
# Process test x using attention
x_e = cos(x + p, z)
x_a = K.softmax(x_e)
s = K.dot(x_a, z)
# Generate new support attention states
qr = K.concatenate([q, r], axis=1)
q, states = self.support_lstm([qr] + states)
# Generate new test attention states
ps = K.concatenate([p, s], axis=1)
p, x_states = self.test_lstm([ps] + x_states)
# Redefine
z = r
# return [x+p, z+q]
return [x + p, xp + q]
def call(self, x, mask=None):
stride_row, stride_col = self.subsample
_, feature_dim, nb_filter = self.W_shape
if self.dim_ordering == 'th':
if K._backend == 'theano':
output = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.nb_row)
slice_col = slice(j * stride_col,
j * stride_col + self.nb_col)
x_flatten = K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim))
output.append(K.dot(x_flatten, self.W[i * self.output_col + j, :, :]))
output = K.concatenate(output, axis=0)
else:
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.nb_row)
slice_col = slice(j * stride_col,
j * stride_col + self.nb_col)
xs.append(K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.W)
output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
output = K.permute_dimensions(output, (2, 3, 0, 1))
elif self.dim_ordering == 'tf':
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.nb_row)
slice_col = slice(j * stride_col,
j * stride_col + self.nb_col)
xs.append(K.reshape(x[:, slice_row, slice_col, :], (1, -1, feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.W)
output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
output = K.permute_dimensions(output, (2, 0, 1, 3))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
if self.bias:
if self.dim_ordering == 'th':
output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col))
elif self.dim_ordering == 'tf':
output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
output = self.activation(output)
return output
def fun(estimation, next_frame, t1, t2, t3):
inputs = concatenate([estimation, next_frame])
A01 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(inputs)
C11 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=2)(A01)
C12 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C11)
C13 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C12)
C14 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C13)
C14 = add([C11, C14])
C21 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C14)
C22 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C21)
C23 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C22)
C24 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C23)
C24 = add([C21, C24])
C31 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=2)(C24)
C32 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C31)
C33 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C32)
C34 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C33)
C34 = add([C31, C34])
C41 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=2)(C34)
C42 = Lambda(lambda x: K.concatenate([x,t1],axis=-1), output_shape=(nx/8,ny/8,512))(C41)
B42 = Conv2D(256, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C42)
C43 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(B42)
C44 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C43)
C45 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C44)
C45 = add([C41, C45])
C51 = transposed_convolution(128, kernel_size=4, l2_reg=l2_reg, strides=2)(C45)
C51 = add([C51, C34])
C52 = Lambda(lambda x: K.concatenate([x,t2],axis=-1),output_shape=(nx/4,ny/4,256))(C51)
B52 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C52)
C53 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(B52)
C54 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C53)
C55 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C54)
C55 = add([C51, C55])
C61 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C55)
C61 = add([C61, C24])
C62 = Lambda(lambda x: K.concatenate([x,t3],axis=-1),output_shape=(nx/2,ny/2,128))(C61)
B62 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C62)
C63 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(B62)
C64 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C63)
C65 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C64)
C65 = add([C61, C65])
C71 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C65)
C72 = convolution(64, kernel_size=4, l2_reg=l2_reg, strides=1)(C71)
out = convolution(1, kernel_size=3, l2_reg=l2_reg, strides=1)(C72)
return out, C43, C53, C63
def step(self, x, states):
h_tild_tm1 = states[0]
B_U = states[1]
B_W = states[2]
if self.consume_less == 'cpu':
x_i = x[:, :self.output_dim]
x_f = x[:, self.output_dim: 2 * self.output_dim]
x_c = x[:, 2 * self.output_dim: 3 * self.output_dim]
x_o = x[:, 3 * self.output_dim: 4 * self.output_dim]
x_new = x[:, 4 * self.output_dim:]
else:
x_i = K.dot(x * B_W[0], self.W_i) + self.b_i
x_f = K.dot(x * B_W[1], self.W_f) + self.b_f
x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
x_new = x
# self.C_tape -> BT, t-1, k
# self.H_tape -> BT, t-1, k
# x -> BT, k
# h_tild_tm1 -> BT, k
if self.H_tape is None:
self.H_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1))
self.C_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1))
# s_t -> BT, t-1, 1
t = K.shape(self.C_tape)[1]
sum1 = K.dot(self.H_tape, self.W_h)
sum2 = K.dot(K.repeat_elements(x_new.dimshuffle((0,'x',1)),t, axis=1), self.W_x)
sum3 = K.dot(K.repeat_elements(h_tild_tm1.dimshuffle((0,'x',1)),t, axis=1), self.W_h_tilde)
tanhed_sum = K.tanh(sum1 + sum2 + sum3)
a_t = K.dot(tanhed_sum, self.v)[:,:,0]
s_t = K.softmax(a_t)
h_tilde_t = T.batched_dot(self.H_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0]
c_tilde_t = T.batched_dot(self.C_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0]
i = self.inner_activation(x_i + K.dot(h_tilde_t * B_U[0], self.U_i))
f = self.inner_activation(x_f + K.dot(h_tilde_t * B_U[1], self.U_f))
c_t = f * c_tilde_t + i * self.activation(x_c + K.dot(h_tilde_t * B_U[2], self.U_c))
o = self.inner_activation(x_o + K.dot(h_tilde_t * B_U[3], self.U_o))
h_t = o * self.activation(c_t)
# Add to Tape
self.C_tape = K.concatenate([self.C_tape, c_t.dimshuffle((0,'x',1))], axis=1)
self.H_tape = K.concatenate([self.H_tape, h_t.dimshuffle((0,'x',1))], axis=1)
return h_t, [h_tilde_t]
def call(self, input):
forward_lstm = LSTM(self.units, dropout=self.dropout,
return_sequences=True, return_state=True)
backward_lstm = LSTM(self.units, dropout=self.dropout,
return_sequences=True, return_state=True, go_backwards=True)
fw_outputs, fw_output, fw_state = forward_lstm(input)
bw_outputs, bw_output, b_state = backward_lstm(input)
bw_outputs = K.reverse(bw_outputs, 1)
# bw_output = bw_state[0]
outputs = K.concatenate([fw_outputs, bw_outputs])
last_output = K.concatenate([fw_output, bw_output])
return [outputs, last_output]
def normals_metric(y_true, y_pred):
y_true = K.variable(y_true)
y_pred = K.variable(y_pred)
y_true = K.expand_dims(y_true,0)
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same')
dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same')
dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same')
mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0
mag_norm_pred = K.sqrt(mag_norm_pred)
grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred,
0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1)
grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred,
1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1)
dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_x = K.abs(dot_term_x)
dot_term_y = K.abs(dot_term_y)
return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def call(self, x, mask=None):
if (self.size == None) or (self.mode == 'sum'):
self.size = int(x.shape[-1])
batch_size, seq_len = K.shape(x)[0], K.shape(x)[1]
position_j = 1. / K.pow(10000., 2 * K.arange(self.size / 2, dtype='float32') / self.size)
position_j = K.expand_dims(position_j, 0)
position_i = K.cumsum(K.ones_like(x[:, :, 0]), 1) - 1 # K.arange不支持变长，只好用这种方法生成
position_i = K.expand_dims(position_i, 2)
position_ij = K.dot(position_i, position_j)
position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2)
if self.mode == 'sum':
return position_ij + x
elif self.mode == 'concat':
return K.concatenate([position_ij, x], 2)
def sparse_amsoftmax_loss(y_true, y_pred, scale=30, margin=0.35):
y_true = K.expand_dims(y_true[:, 0], 1) # 保证y_true的shape=(None, 1)
y_true = K.cast(y_true, 'int32') # 保证y_true的dtype=int32
batch_idxs = K.arange(0, K.shape(y_true)[0])
batch_idxs = K.expand_dims(batch_idxs, 1)
idxs = K.concatenate([batch_idxs, y_true], 1)
y_true_pred = K.tf.gather_nd(y_pred, idxs) # 目标特征，用tf.gather_nd提取出来
y_true_pred = K.expand_dims(y_true_pred, 1)
y_true_pred_margin = y_true_pred - margin # 减去margin
_Z = K.concatenate([y_pred, y_true_pred_margin], 1) # 为计算配分函数
_Z = _Z * scale # 缩放结果，主要因为pred是cos值，范围[-1, 1]
logZ = K.logsumexp(_Z, 1, keepdims=True) # 用logsumexp，保证梯度不消失
logZ = logZ + K.log(1 - K.exp(scale * y_true_pred - logZ)) # 从Z中减去exp(scale * y_true_pred)
return - y_true_pred_margin * scale + logZ
def call(self, X, mask=None):
def reverse(x):
if K.ndim(x) == 2:
x = K.expand_dims(x, -1)
rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
rev = K.squeeze(rev, -1)
else:
rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
return K.permute_dimensions(rev, (1, 0, 2))
Y = self.forward.call(X, mask) # 0,0,0,1,3,6,10
X_rev = reverse(X) # 4,3,2,1,0,0,0
mask_rev = reverse(mask) if mask else None # 1,1,1,1,0,0,0
Y_rev = self.reverse.call(X_rev, mask_rev) # 4,7,9,10,10,10,10
#Fix allignment
if self.return_sequences:
Y_rev = reverse(Y_rev) # 10,10,10,10,9,7,4
if self.merge_mode == 'concat':
return K.concatenate([Y, Y_rev])
elif self.merge_mode == 'sum':
return Y + Y_rev
elif self.merge_mode == 'ave':
return (Y + Y_rev) / 2
elif self.merge_mode == 'mul':
return Y * Y_rev
def test_lambda():
from keras.utils.layer_utils import layer_from_config
Lambda = core.Lambda
layer_test(Lambda,
kwargs={'function': lambda x: x + 1},
input_shape=(3, 2))
# test serialization with function
def f(x):
return x + 1
ld = Lambda(f)
config = ld.get_config()
ld = layer_from_config({'class_name': 'Lambda', 'config': config})
ld = Lambda(lambda x: K.concatenate([K.square(x), x]),
output_shape=lambda s: tuple(list(s)[:-1] + [2 * s[-1]]))
config = ld.get_config()
ld = Lambda.from_config(config)
# test serialization with output_shape function
def f(x):
return K.concatenate([K.square(x), x])
def f_shape(s):
return tuple(list(s)[:-1] + [2 * s[-1]])
ld = Lambda(f, output_shape=f_shape)
config = ld.get_config()
ld = layer_from_config({'class_name': 'Lambda', 'config': config})
def call(self, x, mask=None):
input_shape = K.shape(x)
if self.dim_ordering == 'th':
num_rows = input_shape[2]
num_cols = input_shape[3]
elif self.dim_ordering == 'tf':
num_rows = input_shape[1]
num_cols = input_shape[2]
row_length = [K.cast(num_rows, 'float32') / i for i in self.pool_list]
col_length = [K.cast(num_cols, 'float32') / i for i in self.pool_list]
outputs = []
if self.dim_ordering == 'th':
for pool_num, num_pool_regions in enumerate(self.pool_list):
for ix in range(num_pool_regions):
for jy in range(num_pool_regions):
x1 = ix * col_length[pool_num]
x2 = ix * col_length[pool_num] + col_length[pool_num]
y1 = jy * row_length[pool_num]
y2 = jy * row_length[pool_num] + row_length[pool_num]
x1 = K.cast(K.round(x1), 'int32')
x2 = K.cast(K.round(x2), 'int32')
y1 = K.cast(K.round(y1), 'int32')
y2 = K.cast(K.round(y2), 'int32')
new_shape = [input_shape[0], input_shape[1],
y2 - y1, x2 - x1]
x_crop = x[:, :, y1:y2, x1:x2]
xm = K.reshape(x_crop, new_shape)
pooled_val = K.max(xm, axis=(2, 3))
outputs.append(pooled_val)
elif self.dim_ordering == 'tf':
for pool_num, num_pool_regions in enumerate(self.pool_list):
for ix in range(num_pool_regions):
for jy in range(num_pool_regions):
x1 = ix * col_length[pool_num]
x2 = ix * col_length[pool_num] + col_length[pool_num]
y1 = jy * row_length[pool_num]
y2 = jy * row_length[pool_num] + row_length[pool_num]
x1 = K.cast(K.round(x1), 'int32')
x2 = K.cast(K.round(x2), 'int32')
y1 = K.cast(K.round(y1), 'int32')
y2 = K.cast(K.round(y2), 'int32')
new_shape = [input_shape[0], y2 - y1,
x2 - x1, input_shape[3]]
x_crop = x[:, y1:y2, x1:x2, :]
xm = K.reshape(x_crop, new_shape)
pooled_val = K.max(xm, axis=(1, 2))
outputs.append(pooled_val)
outputs = K.concatenate(outputs)
return outputs
def test_merge():
from keras.layers import Input, merge
from keras.models import Model
# test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
input_shapes = [(3, 2), (3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
# test functional API
for mode in ['sum', 'mul', 'concat', 'ave']:
print(mode)
input_a = Input(shape=input_shapes[0][1:])
input_b = Input(shape=input_shapes[1][1:])
merged = merge([input_a, input_b], mode=mode)
model = Model([input_a, input_b], merged)
model.compile('rmsprop', 'mse')
expected_output_shape = model.get_output_shape_for(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = Model.from_config(config)
model.compile('rmsprop', 'mse')
# test lambda with output_shape lambda
input_a = Input(shape=input_shapes[0][1:])
input_b = Input(shape=input_shapes[1][1:])
merged = merge([input_a, input_b],
mode=lambda tup: K.concatenate([tup[0], tup[1]]),
output_shape=lambda tup: (tup[0][:-1],) + (tup[0][-1] + tup[1][-1],))
expected_output_shape = model.get_output_shape_for(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = Model.from_config(config)
model.compile('rmsprop', 'mse')
# test function with output_shape function
def fn_mode(tup):
x, y = tup
return K.concatenate([x, y])
def fn_output_shape(tup):
s1, s2 = tup
return (s1[:-1],) + (s1[-1] + s2[-1],)
input_a = Input(shape=input_shapes[0][1:])
input_b = Input(shape=input_shapes[1][1:])
merged = merge([input_a, input_b],
mode=fn_mode,
output_shape=fn_output_shape)
expected_output_shape = model.get_output_shape_for(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = Model.from_config(config)
model.compile('rmsprop', 'mse')
def set_batch_function(self, model, input_shape, batch_size, nb_actions, gamma):
input_dim = np.prod(input_shape)
samples = K.placeholder(shape=(batch_size, input_dim * 2 + 3))
S = samples[:, 0 : input_dim]
a = samples[:, input_dim]
a = K.cast(a, '')
r = samples[:, input_dim + 1]
S_prime = samples[:, input_dim + 2 : 2 * input_dim + 2]
game_over = samples[:, 2 * input_dim + 2 : 2 * input_dim + 3]
r = K.reshape(r, (batch_size, 1))
r = K.repeat(r, nb_actions)
r = K.reshape(r, (batch_size, nb_actions))
game_over = K.repeat(game_over, nb_actions)
game_over = K.reshape(game_over, (batch_size, nb_actions))
S = K.reshape(S, (batch_size, ) + input_shape)
S_prime = K.reshape(S_prime, (batch_size, ) + input_shape)
X = K.concatenate([S, S_prime], axis=0)
Y = model(X)
Qsa = K.max(Y[batch_size:], axis=1)
Qsa = K.reshape(Qsa, (batch_size, 1))
Qsa = K.repeat(Qsa, nb_actions)
Qsa = K.reshape(Qsa, (batch_size, nb_actions))
delta = K.reshape(self.one_hot(a, nb_actions), (batch_size, nb_actions))
targets = (1 - delta) * Y[:batch_size] + delta * (r + gamma * (1 - game_over) * Qsa)
self.batch_function = K.function(inputs=[samples], outputs=[S, targets])
def call(self, x):
assert isinstance(x, list)
inp_a, inp_b = x
outp_a = K.l2_normalize(inp_a, -1)
outp_b = K.l2_normalize(inp_b, -1)
alpha = K.batch_dot(outp_b, outp_a, axes=[2, 2])
alpha = K.l2_normalize(alpha, 1)
alpha = K.one_hot(K.argmax(alpha, 1), K.int_shape(inp_a)[1])
hmax = K.batch_dot(alpha, outp_b, axes=[1, 1])
kcon = K.eye(K.int_shape(inp_a)[1], dtype='float32')
m = []
for i in range(self.output_dim):
outp_a = inp_a * self.W[i]
outp_hmax = hmax * self.W[i]
outp_a = K.l2_normalize(outp_a, -1)
outp_hmax = K.l2_normalize(outp_hmax, -1)
outp = K.batch_dot(outp_hmax, outp_a, axes=[2, 2])
outp = K.sum(outp * kcon, -1, keepdims=True)
m.append(outp)
if self.output_dim > 1:
persp = K.concatenate(m, 2)
else:
persp = m
return [persp, persp]
def get_output(self, train=False):
X = self.get_input(train) # 0,0,0,1,2,3,4
mask = self.get_input_mask(train) # 0,0,0,1,1,1,1
def reverse(x):
rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
return K.permute_dimensions(rev, (1, 0, 2))
X_rev = reverse(X) # 4,3,2,1,0,0,0
Y = self.forward(X, mask) # 0,0,0,1,3,6,10
mask_rev = reverse(mask) if mask else None # 1,1,1,1,0,0,0
Y_rev = self.reverse(X_rev, mask_rev) # 4,7,9,10,10,10,10
#Fix allignment
if self.return_sequences:
Y_rev = reverse(Y_rev) # 10,10,10,10,9,7,4
if self.merge_mode == 'concat':
return K.concatenate([Y, Y_rev])
elif self.merge_mode == 'sum':
return Y + Y_rev
elif self.merge_mode == 'ave':
return (Y + Y_rev) / 2
elif self.merge_mode == 'mul':
return Y * Y_rev
def recurrence(y_i, h):
h_permute = K.permute_dimensions(h, [0, 2, 1]) # (batch_size, encoding_dim, input_length)
e = K.l2_normalize(
K.batch_dot(h_permute, s, axes=1), # (batch_size, input_length)
axis=1) # (batch_size, input_length)
# eqn 6
alpha = K.softmax(e) # (batch_size, input_length)
# eqn 5
c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim)
recurrence_result = K.expand_dims(
K.concatenate([c, y_i], axis=1),
dim=1) # (batch_size, 1, 2 * encoding_dim)
expanded_h = Input(shape=(1, 2 * encoding_dim),
name='expanded_h')
gru = Sequential([
GRU(output_dim,
return_sequences=False,
input_shape=(1, 2 * encoding_dim))
])
model = Model(input=[expanded_h],
output=[gru(expanded_h)]) # (batch_size, 1, output_dim)
return model(recurrence_result)
def test_lambda():
layer_test(layers.Lambda,
kwargs={'function': lambda x: x + 1},
input_shape=(3, 2))
layer_test(layers.Lambda,
kwargs={'function': lambda x, a, b: x * a + b,
'arguments': {'a': 0.6, 'b': 0.4}},
input_shape=(3, 2))
# test serialization with function
def f(x):
return x + 1
ld = layers.Lambda(f)
config = ld.get_config()
ld = deserialize_layer({'class_name': 'Lambda', 'config': config})
# test with lambda
ld = layers.Lambda(
lambda x: K.concatenate([K.square(x), x]),
output_shape=lambda s: tuple(list(s)[:-1] + [2 * s[-1]]))
config = ld.get_config()
ld = layers.Lambda.from_config(config)
# test serialization with output_shape function
def f(x):
return K.concatenate([K.square(x), x])
def f_shape(s):
return tuple(list(s)[:-1] + [2 * s[-1]])
ld = layers.Lambda(f, output_shape=f_shape)
config = ld.get_config()
ld = deserialize_layer({'class_name': 'Lambda', 'config': config})
def main(weights_path, base_path, base_file, style_path, style_file,
combo_path, img_width, img_height, iterations):
result_prefix = base_file[:-4] + '_' + style_file[:-4]
base_img_path = base_path + base_file
style_img_path = style_path + style_file
# get tensor representations of images
base_img = K.variable(preprocess_image(base_img_path,
img_width,
img_height))
style_img = K.variable(preprocess_image(style_img_path,
img_width,
img_height))
combo_img = K.placeholder((1, 3, img_width, img_height))
# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_img, style_img, combo_img],
axis=0)
print('Creating painting of {} in the style of {}'.format(base_file[:-4],
style_file[:-4]))
print('Loading model with VGG16 network weights...')
nn = model(weights_path, input_tensor, img_width, img_height)
loss, grads = calc_loss_grad(nn, combo_img, img_width, img_height)
evaluate = Evaluator(loss, grads, combo_img, img_width, img_height)
return optimizer(evaluate, img_width, img_height, combo_path,
result_prefix, iterations=iterations)
def compute_mask(self, inputs, mask=None):
if mask is None or not any([m is not None for m in mask]):
return None
assert hasattr(mask, '__len__') and len(mask) == len(inputs)
if self.mode in ['sum', 'mul', 'ave']:
bool_type = 'bool' if K._BACKEND == 'tensorflow' else 'int32'
masks = [K.cast(m, bool_type) for m in mask if m is not None]
mask = masks[0]
for m in masks[1:]:
mask = mask & m
return mask
elif self.mode in ['concat']:
masks = [K.ones_like(inputs[i][:-1]) if m is None else m for i, m in zip(inputs, mask)]
expanded_dims = [K.expand_dims(m) for m in masks]
concatenated = K.concatenate(expanded_dims, axis=self.concat_axis)
return K.all(concatenated, axis=-1, keepdims=False)
elif self.mode in ['cos', 'dot']:
return None
elif hasattr(self.mode, '__call__'):
if hasattr(self._output_mask, '__call__'):
return self._output_mask(mask)
else:
return self._output_mask
else:
# this should have been caught earlier
raise Exception('Invalid merge mode: {}'.format(self.mode))
def yolo_head(feats, anchors, num_classes, input_shape):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (box_xy + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
box_wh = box_wh * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
return box_xy, box_wh, box_confidence, box_class_probs
def simple_context(X, mask, n=activation_rnn_size):
"""Reduce the input just to its headline part (second half).
For each word in this part it concatenate the output of the previous layer (RNN)
with a weighted average of the outputs of the description part.
In this only the last `rnn_size - activation_rnn_size` are used from each output.
The first `activation_rnn_size` output is used to computer the weights for the averaging.
"""
desc, head = X[:, :maxlend, :], X[:, maxlend:, :]
head_activations, head_words = head[:, :, :n], head[:, :, n:]
desc_activations, desc_words = desc[:, :, :n], desc[:, :, n:]
# RTFM http://deeplearning.net/software/theano/library/tensor/basic.html#theano.tensor.batched_tensordot
# activation for every head word and every desc word
activation_energies = K.batch_dot(head_activations, desc_activations, axes=(2, 2))
# make sure we dont use description words that are masked out
activation_energies = activation_energies + -1e20 * K.expand_dims(
1. - K.cast(mask[:, :maxlend], 'float32'), 1)
# for every head word compute weights for every desc word
activation_energies = K.reshape(activation_energies, (-1, maxlend))
activation_weights = K.softmax(activation_energies)
activation_weights = K.reshape(activation_weights, (-1, maxlenh, maxlend))
# for every head word compute weighted average of desc words
desc_avg_word = K.batch_dot(activation_weights, desc_words, axes=(2, 1))
return K.concatenate((desc_avg_word, head_words))
def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
global add_tables
nb_filter = output_shape[0]
conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x)
conv = BatchNormalization(axis=1)(conv)
conv = Activation("relu")(conv)
conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv)
conv = BatchNormalization(axis=1)(conv)
if strides[0] >= 2:
x = AveragePooling2D(strides)(x)
if (output_shape[0] - input_shape[0]) > 0:
pad_shape = (1,
output_shape[0] - input_shape[0],
output_shape[1],
output_shape[2])
padding = K.ones(pad_shape)
padding = K.repeat_elements(padding, K.shape(x)[0], axis=0)
x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
output_shape=output_shape)(x)
_death_rate = K.variable(death_rate)
scale = K.ones_like(conv) - _death_rate
conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
output_shape=output_shape)(conv)
out = merge([conv, x], mode="sum")
out = Activation("relu")(out)
gate = K.variable(1, dtype="uint8")
add_tables += [{"death_rate": _death_rate, "gate": gate}]
return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
output_shape=output_shape)([out, x])
def get_output(self, train=False):
print "Input Shape", self.input_shape
print "ConvolutionDNASequenceBinding", self.output_shape
X = self.get_input(train)
if self.use_three_base_encoding:
X_fwd = X[:,1:,:,:]
X_rc = X[:,:3,:,:]
else:
X_fwd = X
X_rc = X
print self.W
print self.b
if self.W[1] is not None:
W = self.W[0][self.W[1],:,:,:]
else:
W = self.W[0]
if self.b[1] is not None:
b = self.b[0][self.b[1]]
else:
b = self.b[0]
fwd_rv = K.conv2d(X_fwd, W, border_mode='valid') \
+ K.reshape(b, (1, self.nb_motifs, 1, 1))
rc_rv = K.conv2d(X_rc, W[:,::-1,:,::-1], border_mode='valid') \
+ K.reshape(b, (1, self.nb_motifs, 1, 1))
rv = K.concatenate((fwd_rv, rc_rv), axis=2)
#return rv.dimshuffle((0,3,2,1))
return rv # K.permute_dimensions(rv, (0,3,2,1))
def get_output(self, train):
x = self.get_input(train)
x -= K.mean(x, axis=1, keepdims=True)
x = K.l2_normalize(x, axis=1)
pos = K.relu(x)
neg = K.relu(-x)
return K.concatenate([pos, neg], axis=1)
return x
# endregion
# region load the pretrained VGG19
print('loading model...')
target_image = K.constant(preprocess_image(target_image_path))
style_reference_image = K.constant(
preprocess_image(style_reference_image_path))
# This placeholder will contain our generated image
combination_image = K.placeholder((1, img_height, img_width, 3))
# We combine the 3 images into a single batch
input_tensor = K.concatenate(
[target_image, style_reference_image, combination_image], axis=0)
# We build the VGG19 network with our batch of 3 images as input.
# The model will be loaded with pre-trained ImageNet weights.
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
print('Model loaded.')
# endregion
# region content loss
def content_loss(base, combination):
return K.sum(K.square((combination - base)))
yield [x, y], z
x, y = [], []
_ = 0
#CBOW输入
input_words = Input(shape=(window * 2, ), dtype='int32')
input_vecs = Embedding(nb_word, word_size, name='word2vec')(input_words)
input_vecs_sum = Lambda(lambda x: K.sum(x, axis=1))(
input_vecs) #CBOW模型，直接将上下文词向量求和
#构造随机负样本，与目标组成抽样
target_word = Input(shape=(1, ), dtype='int32')
negatives = Lambda(lambda x: K.random_uniform(
(K.shape(x)[0], nb_negative), 0, nb_word, 'int32'))(target_word)
samples = Lambda(lambda x: K.concatenate(x))(
[target_word, negatives]) #构造抽样，负样本随机抽。负样本也可能抽到正样本，但概率小。
#只在抽样内做Dense和softmax
softmax_weights = Embedding(nb_word, word_size, name='W')(samples)
softmax_biases = Embedding(nb_word, 1, name='b')(samples)
softmax = Lambda(lambda x: K.softmax(
(K.batch_dot(x[0], K.expand_dims(x[1], 2)) + x[2])[:, :, 0]))(
[softmax_weights, input_vecs_sum,
softmax_biases]) #用Embedding层存参数，用K后端实现矩阵乘法，以此复现Dense层的功能
#留意到，我们构造抽样时，把目标放在了第一位，也就是说，softmax的目标id总是0，这可以从data_generator中的z变量的写法可以看出
model = Model(inputs=[input_words, target_word], outputs=softmax)
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
def lnln_model_flip(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=True):
reg_val = 1
v_leak = 0
v_exc = 60
v_inh = -30
g_leak = 1
filter_shape[1] = 1
assert (np.mod(num_filter, 2) == 0)
num_filter = int(num_filter / 2)
pad_x = int((filter_shape[1] - 1)) + 2
pad_t = int((filter_shape[0] - 1))
# Define the input as a tensor with shape input_shape
image_in = Input(input_shape)
s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in)
s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in)
s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in)
g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)
g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)
g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)
g2_both = g2(s2)
g1_1 = g1(s1)
g1_2 = g1(s3)
g3_1 = g3(s3)
g3_2 = g3(s1)
expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1))
squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1))
g2_both = expand_last(g2_both)
g1_1 = expand_last(g1_1)
g1_2 = expand_last(g1_2)
g3_1 = expand_last(g3_1)
g3_2 = expand_last(g3_2)
numerator_in_1 = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1_1, g2_both, g3_1])
numerator_in_2 = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1_2, g2_both, g3_2])
numerator_comb = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)
vm_1 = numerator_comb(numerator_in_1)
vm_2 = numerator_comb(numerator_in_2)
vm_1 = squeeze_last(vm_1)
vm_2 = squeeze_last(vm_2)
bias_layer = BiasLayer()
vm_1_bias = bias_layer(vm_1)
vm_2_bias = bias_layer(vm_2)
vm_1_rect = Lambda(lambda lam: K.relu(lam))(vm_1_bias)
vm_2_rect = Lambda(lambda lam: K.relu(lam))(vm_2_bias)
vm = subtract([vm_1_rect, vm_2_rect])
if sum_over_space:
conv_x_size = vm.get_shape().as_list()[2]
else:
conv_x_size = 1
combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)(vm)
# Create model
model = Model(inputs=image_in, outputs=combine_filters, name='lnln_model_flip')
return model, pad_x, pad_t
def zeropad(x):
y = K.zeros_like(x)
return K.concatenate([x, y], axis=2)
def custom_loss(y_true, y_pred):
return -K.mean(
self.critic(
K.concatenate(
[input_tensor, self.actor(input_tensor)], axis=1)))
def step(self, x, states, training=None):
h_tm1 = states[0]
c_tm1 = states[1]
x_seq = states[2]
# repeat the hidden state to the length of the sequence
_htm = K.repeat(h_tm1, self.time_step_e) #(batch,time_step,units)
# concatenate a(previus output lstm) + hidden state
concatenate = K.concatenate(
[_htm, x_seq], axis=-1) #(batch,time_step,h_units+x_seq_units)
# now multiplty the weight matrix with the repeated hidden state
# apply the a dense layer over the time dimension of the sequence
# do it here because it doesn't depend on any previous steps
# thefore we can save computation time:
dot_dense = time_distributed_dense(
concatenate,
self.W_cx,
b=self.b_cx,
input_dim=self.input_dim_e + self.units,
timesteps=self.time_step_e,
output_dim=self.atten_units) #(samples,timestep,atten_units)
# we need to supply the full sequence of inputs to step (as the attention_vector)
# calculate the attention probabilities
# this relates how much other timesteps contributed to this one.
et = K.dot(
K.relu(dot_dense), #(batch,time_step,atten_units)
K.expand_dims(self.C_cx))
at = K.exp(et) #(batch,time_step,1)
at_sum = K.cast(K.sum(at, axis=1) + K.epsilon(),
K.floatx()) #(batch,1)
at_sum_repeated = K.repeat(at_sum,
self.time_step_e) #(batch,time_step,1)
at /= at_sum_repeated # vector of size (batchsize, time_steps, 1)
# calculate the context vector
context = K.squeeze(K.batch_dot(at, x_seq, axes=1),
axis=1) #(batchsize,input_dim)
if 0 < self.dropout < 1 and self._dropout_mask is None:
self._dropout_mask = _generate_dropout_mask(K.ones_like(context),
self.dropout,
training=training,
count=4)
if (0 < self.recurrent_dropout < 1
and self._recurrent_dropout_mask is None):
self._recurrent_dropout_mask = _generate_dropout_mask(
K.ones_like(states[0]),
self.recurrent_dropout,
training=training,
count=4)
# dropout matrices for input units
B_W = self._dropout_mask
# dropout matrices for recurrent units
B_U = self._recurrent_dropout_mask
# ~~~> calculate new hidden state
yhat_i = K.dot(x, self.V_i) #(batchsize,units)
yhat_f = K.dot(x, self.V_f) #(batchsize,units)
yhat_c = K.dot(x, self.V_c) #(batchsize,units)
yhat_o = K.dot(x, self.V_o) #(batchsize,units)
if 0 < self.dropout < 1.:
x_i = K.dot(context * B_W[0],
self.W_i) + self.b_i #(batchsize,units)
x_f = K.dot(context * B_W[1],
self.W_f) + self.b_f #(batchsize,units)
x_c = K.dot(context * B_W[2],
self.W_c) + self.b_c #(batchsize,units)
x_o = K.dot(context * B_W[3],
self.W_o) + self.b_o #(batchsize,units)
else:
x_i = K.dot(context, self.W_i) + self.b_i #(batchsize,units)
x_f = K.dot(context, self.W_f) + self.b_f #(batchsize,units)
x_c = K.dot(context, self.W_c) + self.b_c #(batchsize,units)
x_o = K.dot(context, self.W_o) + self.b_o #(batchsize,units)
if 0 < self.recurrent_dropout < 1.:
h_tm1_i = K.dot(h_tm1 * B_U[0], self.U_i) #(batchsize,units)
h_tm1_f = K.dot(h_tm1 * B_U[1], self.U_f) #(batchsize,units)
h_tm1_c = K.dot(h_tm1 * B_U[2], self.U_c) #(batchsize,units)
h_tm1_o = K.dot(h_tm1 * B_U[3], self.U_o) #(batchsize,units)
else:
h_tm1_i = K.dot(h_tm1, self.U_i) #(batchsize,units)
h_tm1_f = K.dot(h_tm1, self.U_f) #(batchsize,units)
h_tm1_c = K.dot(h_tm1, self.U_c) #(batchsize,units)
h_tm1_o = K.dot(h_tm1, self.U_o) #(batchsize,units)
i = self.recurrent_activation(x_i + h_tm1_i +
yhat_i) #(batchsize,units)
f = self.recurrent_activation(x_f + h_tm1_f +
yhat_f) #(batchsize,units)
o = self.recurrent_activation(x_o + h_tm1_o +
yhat_o) #(batchsize,units)
c_ = self.activation(x_c + h_tm1_c + yhat_c) #(batchsize,units)
c = f * c_tm1 + i * c_
h = o * self.activation(c) #(batchsize,units)
if 0 < self.dropout + self.recurrent_dropout:
if training is None:
h._uses_learning_phase = True
#apply maxout layer with dropout
maxout = self.max_out(inputs=K.dot(h, self.U_p), num_units=self.gmax)
drop = Dropout(0.3)(maxout)
#apply softmax
_y_hat = activations.softmax(K.dot(drop, self.M_p) + self.b_p)
if self.return_probabilities:
return at, [h, c]
else:
return _y_hat, [h, c]
def call(self, x, mask=None):
input_shape = K.shape(x)
if self.dim_ordering == 'th':
num_rows = input_shape[2]
num_cols = input_shape[3]
elif self.dim_ordering == 'tf':
num_rows = input_shape[1]
num_cols = input_shape[2]
row_length = [K.cast(num_rows, 'float32') / i for i in self.pool_list]
col_length = [K.cast(num_cols, 'float32') / i for i in self.pool_list]
outputs = []
if self.dim_ordering == 'th':
for pool_num, num_pool_regions in enumerate(self.pool_list):
for jy in range(num_pool_regions):
for ix in range(num_pool_regions):
x1 = ix * col_length[pool_num]
x2 = ix * col_length[pool_num] + col_length[pool_num]
y1 = jy * row_length[pool_num]
y2 = jy * row_length[pool_num] + row_length[pool_num]
x1 = K.cast(K.round(x1), 'int32')
x2 = K.cast(K.round(x2), 'int32')
y1 = K.cast(K.round(y1), 'int32')
y2 = K.cast(K.round(y2), 'int32')
new_shape = [input_shape[0], input_shape[1],
y2 - y1, x2 - x1]
x_crop = x[:, :, y1:y2, x1:x2]
xm = K.reshape(x_crop, new_shape)
pooled_val = K.max(xm, axis=(2, 3))
outputs.append(pooled_val)
elif self.dim_ordering == 'tf':
for pool_num, num_pool_regions in enumerate(self.pool_list):
for jy in range(num_pool_regions):
for ix in range(num_pool_regions):
x1 = ix * col_length[pool_num]
x2 = ix * col_length[pool_num] + col_length[pool_num]
y1 = jy * row_length[pool_num]
y2 = jy * row_length[pool_num] + row_length[pool_num]
x1 = K.cast(K.round(x1), 'int32')
x2 = K.cast(K.round(x2), 'int32')
y1 = K.cast(K.round(y1), 'int32')
y2 = K.cast(K.round(y2), 'int32')
new_shape = [input_shape[0], y2 - y1,
x2 - x1, input_shape[3]]
x_crop = x[:, y1:y2, x1:x2, :]
xm = K.reshape(x_crop, new_shape)
pooled_val = K.max(xm, axis=(1, 2))
outputs.append(pooled_val)
if self.dim_ordering == 'th':
outputs = K.concatenate(outputs)
elif self.dim_ordering == 'tf':
#outputs = K.concatenate(outputs,axis = 1)
outputs = K.concatenate(outputs)
#outputs = K.reshape(outputs,(len(self.pool_list),self.num_outputs_per_channel,input_shape[0],input_shape[1]))
#outputs = K.permute_dimensions(outputs,(3,1,0,2))
#outputs = K.reshape(outputs,(input_shape[0], self.num_outputs_per_channel * self.nb_channels))
return outputs
# this will contain our generated image
if K.image_dim_ordering() == 'th':
combination_image = K.placeholder((1, 3, img_width, img_height))
else:
combination_image = K.placeholder((1, img_width, img_height, 3))
image_tensors = [base_image]
for style_image_tensor in style_reference_images:
image_tensors.append(style_image_tensor)
image_tensors.append(combination_image)
nb_tensors = len(image_tensors)
nb_style_images = nb_tensors - 2 # Content and Output image not considered
# combine the various images into a single Keras tensor
input_tensor = K.concatenate(image_tensors, axis=0)
if K.image_dim_ordering() == "th":
shape = (nb_tensors, 3, img_width, img_height)
else:
shape = (nb_tensors, img_width, img_height, 3)
ip = Input(tensor=input_tensor, batch_shape=shape)
# build the VGG16 network with our 3 images as input
x = Convolution2D(64, (3, 3),
activation='relu',
name='conv1_1',
padding='same')(ip)
x = Convolution2D(64, (3, 3),
activation='relu',
def concat_tensors(tensors, axis=-1):
return K.concatenate([K.expand_dims(t, axis=axis) for t in tensors])
def call(self, inputs):
if self.r_num == 1:
# if there is no routing (and this is so when r_num is 1 and all c are equal)
# then this is a common convolution
outputs = K.conv2d(
K.reshape(inputs,
(-1, self.h_i, self.w_i, self.ch_i * self.n_i)),
K.reshape(
self.w, self.kernel_size +
(self.ch_i * self.n_i, self.ch_j * self.n_j)),
data_format='channels_last',
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate)
outputs = squeeze(
K.reshape(outputs,
((-1, self.h_j, self.w_j, self.ch_j, self.n_j))))
else:
bt = K.shape(inputs)[0]
ksz = self.kernel_size[0] * self.kernel_size[1]
xr = K.reshape(inputs,
(-1, self.h_i, self.w_i, self.ch_i * self.n_i))
pt = tf.extract_image_patches(xr, (1, ) + self.kernel_size + (1, ),
(1, ) + self.strides + (1, ), (
1,
1,
1,
1,
), 'VALID')
pt = K.reshape(pt, (-1, ksz * self.ch_i, self.n_i))
wr = K.reshape(self.w,
(ksz * self.ch_i, self.n_i, self.ch_j * self.n_j))
global useGPU
# it sometimes works faster on GPU when batch is devided into two parts
# bp = K.expand_dims(bt // 2, axis=0) if useGPU else K.constant([2], dtype=tf.int32)
bp = K.expand_dims(bt // 1, axis=0) if useGPU else K.constant(
[2], dtype=tf.int32)
if self.strides != (1, 1):
zr_shape = K.concatenate([
bp,
K.constant([self.w_j, ksz * self.ch_i * self.ch_j],
dtype=tf.int32)
])
zr = tf.zeros(shape=zr_shape)
zc_shape = K.concatenate([
bp,
K.constant([self.ah_j, ksz * self.ch_i * self.ch_j],
dtype=tf.int32)
])
zc = tf.zeros(shape=zc_shape)
def rt(ptb):
ptb = K.reshape(ptb, (-1, ksz * self.ch_i, self.n_i))
if useGPU:
ub = tf.einsum('bin,inj->bij', ptb, wr)
else:
ul = []
for i in range(ksz * self.ch_i):
ul.append(K.dot(ptb[:, i], wr[i]))
ub = K.stack(ul, axis=1)
#b = tf.constant_initializer(0.)((bp, self.h_i*self.w_i*self.ch_i,
# ksz * self.ch_j))
b = 0.0
j_all = self.h_j * self.w_j * self.ch_j
j_add = j_all - ksz * self.ch_j
for r in range(self.r_num):
ex = K.exp(b * self.b_alphas[r])
if r > 0:
c = ex / (
(K.sum(ex, axis=-1, keepdims=True) + K.epsilon()) +
j_add) * j_all
c = K.reshape(c, (-1, self.h_i, self.w_i,
self.ch_i * ksz * self.ch_j))
c = K.stop_gradient(c)
pc = tf.extract_image_patches(
c, (1, ) + self.kernel_size + (1, ),
(1, ) + self.strides + (1, ), (
1,
1,
1,
1,
), 'VALID')
pc = K.reshape(pc, (-1, self.h_j, self.w_j, ksz,
self.ch_i, self.kernel_size[0] *
self.kernel_size[1], self.ch_j))
pcl = []
for n in range(ksz):
pcl.append(
pc[:, :, :, n, :,
self.kernel_size[0] * self.kernel_size[1] -
1 - n])
pcc = K.stack(pcl, axis=3)
if useGPU:
pcc = K.reshape(pcc,
(-1, self.h_j * self.w_j * ksz *
self.ch_i * self.ch_j, 1))
ub = K.reshape(ub,
(-1, self.h_j * self.w_j * ksz *
self.ch_i * self.ch_j, self.n_j))
cu = pcc * ub
else:
pcc = K.reshape(pcc, (-1, 1))
ub = K.reshape(ub, (-1, self.n_j, 1))
cul = []
for n in range(self.n_j):
cul.append(ub[:, n] * pcc)
cu = K.stack(cul, axis=-2)
else:
cu = ub
cu = K.reshape(cu, (-1, self.h_j * self.w_j,
ksz * self.ch_i, self.ch_j, self.n_j))
s = K.sum(cu, axis=-3)
v = squeeze(s)
if r == self.r_num - 1:
break
v = K.stop_gradient(v)
ubr = K.reshape(K.stop_gradient(ub),
(-1, self.h_j * self.w_j, ksz * self.ch_i,
self.ch_j, self.n_j))
if True:
#if useGPU:
a = tf.einsum('bjck,bjick->bjic', v, ubr)
else:
al = []
for i in range(ksz * self.ch_i):
al.append(
K.batch_dot(
K.reshape(ubr[:, :, i],
(-1, self.h_j * self.w_j *
self.ch_j, 1, self.n_j)),
K.reshape(v, (-1, self.h_j * self.w_j *
self.ch_j, self.n_j, 1))))
a = K.stack(al, axis=1)
a = K.reshape(a, (-1, ksz * self.ch_i,
self.h_j * self.w_j, self.ch_j))
a = K.permute_dimensions(a, [0, 2, 1, 3])
ph, pw = 2, 2
a = K.reshape(
a,
(-1, self.h_j, self.w_j, ksz * self.ch_i * self.ch_j))
if self.strides == (1, 1):
aa = a
else:
rl = []
for r in range(self.ah_j):
rl.append(zr if r % ph else a[:, r // ph])
rs = K.stack(rl, axis=1)
cl = []
for c in range(self.aw_j):
cl.append(zc if c % pw else rs[:, :, c // pw])
aa = K.stack(cl, axis=-2)
aa = K.spatial_2d_padding(aa, ((ph, ph), (pw, pw)),
data_format='channels_last')
pa = tf.extract_image_patches(
aa,
(1, ) + self.kernel_size + (1, ),
(
1,
1,
1,
1,
), #(1,)+strides+(1,),
(
1,
1,
1,
1,
),
'VALID')
pa = K.reshape(pa, (-1, self.h_i * self.w_i, ksz, ksz,
self.ch_i, self.ch_j))
pal = []
for n in range(ksz):
pal.append(pa[:, :, n, ksz - 1 - n])
paa = K.stack(pal, axis=3)
paa = K.reshape(
paa,
(-1, self.h_i * self.w_i * self.ch_i, ksz * self.ch_j))
b = b + paa
return v
v = tf.map_fn(rt,
K.reshape(pt, (-1, bp[0], self.h_j, self.w_j,
ksz * self.ch_i, self.n_i)),
parallel_iterations=100,
back_prop=True,
infer_shape=False)
outputs = v
outputs = K.reshape(outputs,
(-1, self.h_j, self.w_j, self.ch_j, self.n_j))
return outputs
# Create tensor variables for images
if K.image_dim_ordering() == 'th':
shape = (1, nb_colors, img_nrows, img_ncols)
else:
shape = (1, img_nrows, img_ncols, nb_colors)
style_image = K.variable(preprocess_image(style_img_path))
target_image = K.placeholder(shape=shape)
if use_content_img:
content_image = K.variable(preprocess_image(content_img_path))
else:
content_image = K.zeros(shape=shape)
images = K.concatenate([style_image, target_image, content_image], axis=0)
# Create tensor variables for masks
raw_style_mask, raw_target_mask = load_mask_labels()
style_mask = K.variable(raw_style_mask.astype("float32"))
target_mask = K.variable(raw_target_mask.astype("float32"))
masks = K.concatenate([style_mask, target_mask], axis=0)
# index constants for images and tasks variables
STYLE, TARGET, CONTENT = 0, 1, 2
# Build image model, mask model and use layer outputs as features
# image model as VGG19
image_model = vgg19.VGG19(include_top=False, input_tensor=images)
# mask model as a series of pooling
#for target user
h_i = decoder_pred
h_all = h_i
if cfg.use_embedding_AME:
W_i = W_i_pool[-1]
u_i_list.append(W_i(h_all))
V_i = V_i_pool[-1]
u_si_list.append(V_i(h_i))
else:
u_i_list.append(h_all)
u_si_list.append(h_i)
assert len(u_i_list)==num_user
assert len(u_si_list)==num_user
u_i_list = Lambda(lambda x: K.concatenate(x, axis=1))(u_i_list)#(batch,34,256)
u_si_list = Lambda(lambda x: K.concatenate(x, axis=1))(u_si_list)#(batch,34,256)
alpha_list = ui_usi_similarity(u_i_list,u_si_list)
outputs = merge_experts_contribution(alpha_list,concat_outputs)
all_outputs.append(outputs)
inputs = outputs
## Concatenate all predictions
decoder_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs)
model = Model([encoder_inputs, others_inputs, decoder_inputs], decoder_outputs)
model.compile(optimizer='Adam', loss='mean_squared_error',metrics=['accuracy'])
def call(self, inputs, mask=None):
'''
:param inputs: a list of tensor of length not larger than 2, or a memory tensor of size BxTXD1.
If a list, the first entry is memory, and the second one is query tensor of size BxD2 if any
:param mask: the masking entry will be directly discarded
:return: a tensor of size BxD1, weighted summing along the sequence dimension
'''
query = None
if isinstance(inputs, list):
memory = inputs[0]
if len(inputs) > 1:
query = inputs[1]
elif len(inputs) > 2:
raise ValueError('inputs length should not be larger than 2')
if isinstance(mask, list):
mask = mask[0]
else:
memory = inputs
input_shape = K.int_shape(memory)
if len(input_shape) > 3:
input_length = input_shape[1]
memory = K.reshape(memory, (-1, ) + input_shape[2:])
if mask is not None:
mask = K.reshape(mask, (-1, ) + input_shape[2:-1])
if query is not None:
raise ValueError('query can be not supported')
last = memory[:, -1, :]
memory = memory[:, :-1, :]
if query is None:
query = last
else:
query = K.concatenate([query, last], axis=-1)
if self.method is None:
if len(input_shape) > 3:
output_shape = K.int_shape(last)
return K.reshape(last, (-1, input_shape[1], output_shape[-1]))
else:
return last
elif self.method == 'cba':
hidden = K.dot(memory, self.Wh) + K.expand_dims(
K.dot(query, self.Wq), 1)
hidden = K.tanh(hidden)
s = K.squeeze(K.dot(hidden, self.v), -1)
elif self.method == 'ga':
s = K.sum(K.expand_dims(K.dot(query, self.Wq), 1) * memory,
axis=-1)
else:
s = K.squeeze(K.dot(memory, self.v), -1)
s = K.softmax(s)
if mask is not None:
mask = mask[:, :-1]
s *= K.cast(mask, dtype='float32')
sum_by_time = K.sum(s, axis=-1, keepdims=True)
s = s / (sum_by_time + K.epsilon())
#return [K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1), s]
result = K.concatenate(
[K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1)
if len(input_shape) > 3:
output_shape = K.int_shape(result)
return K.reshape(result, (-1, input_shape[1], output_shape[-1]))
else:
return result
content_array[:, :, :, 0] -= 103.939
content_array[:, :, :, 1] -= 116.779
content_array[:, :, :, 2] -= 123.68
style_array[:, :, :, 0] -= 103.939
style_array[:, :, :, 1] -= 116.779
style_array[:, :, :, 2] -= 123.68
height = 512
width = 512
content_image = backend.variable(content_array)
style_image = backend.variable(style_array)
combination_image = backend.placeholder((1, height, width, 3))
input_tensor = backend.concatenate(
[content_image, style_image, combination_image], axis=0)
model = VGG16(input_tensor=input_tensor, weights='imagenet', include_top=False)
model.summary()
content_weight = backend.variable(0.05)
style_weight = backend.variable(50.0)
total_variation_weight = backend.variable(1.0)
layers = dict([(layer.name, layer.output) for layer in model.layers])
print(layers)
loss = backend.variable(0.)
def content_loss(content, combination):
def build(self, input_shape):
self.T = input_shape[1]
if self.first:
self.input_dim = input_shape[2]
else:
self.input_dim = input_shape[2] - 1
self.input_spec = [InputSpec(shape=input_shape)]
self.states = [None, None, None]
self.W_i = self.init((self.input_dim, self.output_dim),
name='{}_W_i'.format(self.name))
self.U_i = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_i'.format(self.name))
self.b_i = K.zeros((self.output_dim, ),
name='{}_b_i'.format(self.name))
self.W_f = self.init((self.input_dim, self.output_dim),
name='{}_W_f'.format(self.name))
self.U_f = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_f'.format(self.name))
self.b_f = self.forget_bias_init((self.output_dim, ),
name='{}_b_f'.format(self.name))
self.W_c = self.init((self.input_dim, self.output_dim),
name='{}_W_c'.format(self.name))
self.U_c = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_c'.format(self.name))
self.b_c = K.zeros((self.output_dim, ),
name='{}_b_c'.format(self.name))
self.W_o = self.init((self.input_dim, self.output_dim),
name='{}_W_o'.format(self.name))
self.U_o = self.inner_init((self.output_dim, self.output_dim),
name='{}_U_o'.format(self.name))
self.b_o = K.zeros((self.output_dim, ),
name='{}_b_o'.format(self.name))
self.W_s = self.init((self.input_dim, 1),
name='{}_W_s'.format(self.name))
self.U_s = self.inner_init((self.output_dim, 1),
name='{}_U_s'.format(self.name))
self.b_s = K.zeros((1, ), name='{}_b_s'.format(self.name))
self.regularizers = []
if self.W_regularizer:
self.W_regularizer.set_param(
K.concatenate(
[self.W_i, self.W_f, self.W_c, self.W_o, self.W_s]))
self.regularizers.append(self.W_regularizer)
if self.U_regularizer:
self.U_regularizer.set_param(
K.concatenate(
[self.U_i, self.U_f, self.U_c, self.U_o, self.U_s]))
self.regularizers.append(self.U_regularizer)
if self.b_regularizer:
self.b_regularizer.set_param(
K.concatenate(
[self.b_i, self.b_f, self.b_c, self.b_o, self.b_s]))
self.regularizers.append(self.b_regularizer)
self.trainable_weights = [
self.W_i, self.U_i, self.b_i, self.W_c, self.U_c, self.b_c,
self.W_f, self.U_f, self.b_f, self.W_o, self.U_o, self.b_o,
self.W_s, self.U_s, self.b_s
]
def yolo_loss(args,
anchors,
num_classes,
rescore_confidence=False,
print_loss=False):
"""YOLO localization loss function.
Parameters
----------
yolo_output : tensor
Final convolutional layer features.
true_boxes : tensor
Ground truth boxes tensor with shape [batch, num_true_boxes, 5]
containing box x_center, y_center, width, height, and class.
detectors_mask : array
0/1 mask for detector positions where there is a matching ground truth.
matching_true_boxes : array
Corresponding ground truth boxes for positive detector positions.
Already adjusted for conv height and width.
anchors : tensor
Anchor boxes for model.
num_classes : int
Number of object classes.
rescore_confidence : bool, default=False
If true then set confidence target to IOU of best predicted box with
the closest matching ground truth box.
print_loss : bool, default=False
If True then use a tf.Print() to print the loss components.
Returns
-------
mean_loss : float
mean localization loss across minibatch
"""
(yolo_output, true_boxes, detectors_mask, matching_true_boxes) = args
num_anchors = len(anchors)
object_scale = 5
no_object_scale = 1
class_scale = 1
coordinates_scale = 1
pred_xy, pred_wh, pred_confidence, pred_class_prob = yolo_head(
yolo_output, anchors, num_classes)
# Unadjusted box predictions for loss.
# TODO: Remove extra computation shared with yolo_head.
yolo_output_shape = K.shape(yolo_output)
feats = K.reshape(yolo_output, [
-1, yolo_output_shape[1], yolo_output_shape[2], num_anchors,
num_classes + 5
])
pred_boxes = K.concatenate(
(K.sigmoid(feats[..., 0:2]), feats[..., 2:4]), axis=-1)
# TODO: Adjust predictions by image width/height for non-square images?
# IOUs may be off due to different aspect ratio.
# Expand pred x,y,w,h to allow comparison with ground truth.
# batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params
pred_xy = K.expand_dims(pred_xy, 4)
pred_wh = K.expand_dims(pred_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
true_boxes_shape = K.shape(true_boxes)
# batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params
true_boxes = K.reshape(true_boxes, [
true_boxes_shape[0], 1, 1, 1, true_boxes_shape[1], true_boxes_shape[2]
])
true_xy = true_boxes[..., 0:2]
true_wh = true_boxes[..., 2:4]
# Find IOU of each predicted box with each ground truth box.
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
intersect_mins = K.maximum(pred_mins, true_mins)
intersect_maxes = K.minimum(pred_maxes, true_maxes)
intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = intersect_areas / union_areas
# Best IOUs for each location.
best_ious = K.max(iou_scores, axis=4) # Best IOU scores.
best_ious = K.expand_dims(best_ious)
# A detector has found an object if IOU > thresh for some true box.
object_detections = K.cast(best_ious > 0.6, K.dtype(best_ious))
# TODO: Darknet region training includes extra coordinate loss for early
# training steps to encourage predictions to match anchor priors.
# Determine confidence weights from object and no_object weights.
# NOTE: YOLO does not use binary cross-entropy here.
no_object_weights = (no_object_scale * (1 - object_detections) *
(1 - detectors_mask))
no_objects_loss = no_object_weights * K.square(-pred_confidence)
if rescore_confidence:
objects_loss = (object_scale * detectors_mask *
K.square(best_ious - pred_confidence))
else:
objects_loss = (object_scale * detectors_mask *
K.square(1 - pred_confidence))
confidence_loss = objects_loss + no_objects_loss
# Classification loss for matching detections.
# NOTE: YOLO does not use categorical cross-entropy loss here.
matching_classes = K.cast(matching_true_boxes[..., 4], 'int32')
matching_classes = K.one_hot(matching_classes, num_classes)
classification_loss = (class_scale * detectors_mask *
K.square(matching_classes - pred_class_prob))
# Coordinate loss for matching detection boxes.
matching_boxes = matching_true_boxes[..., 0:4]
coordinates_loss = (coordinates_scale * detectors_mask *
K.square(matching_boxes - pred_boxes))
confidence_loss_sum = K.sum(confidence_loss)
classification_loss_sum = K.sum(classification_loss)
coordinates_loss_sum = K.sum(coordinates_loss)
total_loss = 0.5 * (
confidence_loss_sum + classification_loss_sum + coordinates_loss_sum)
if print_loss:
total_loss = tf.Print(
total_loss, [
total_loss, confidence_loss_sum, classification_loss_sum,
coordinates_loss_sum
],
message='yolo_loss, conf_loss, class_loss, box_coord_loss:')
return total_loss
def attributes_update(self, attributes, depth, graph, original_graph,
bonds):
'''Given the current attributes, the current depth, and the graph that the attributes
are based on, this function will update the 2D attributes tensor'''
############# GET NEW ATTRIBUTE MATRIX #########################
# New pre-activated attribute matrix v = M_i,j,: x ones((N_atom, 1)) -> (N_atom, N_features)
# as long as dimensions are appropriately shuffled
shuffled_graph = graph.copy().dimshuffle(
(2, 0, 1)) # (N_feature x N_atom x N_atom)
shuffled_graph.name = 'shuffled_graph'
ones_vec = K.ones_like(attributes[:, 0]) # (N_atom x 1)
ones_vec.name = 'ones_vec'
(new_preactivated_attributes, updates) = theano.scan(
lambda x: K.dot(x, ones_vec),
sequences=shuffled_graph) # (N_features x N_atom)
# Need to pass through an activation function still
# Final attribute = bond flag = is not part of W_inner or b_inner
(new_attributes,
updates) = theano.scan(lambda x: self.activation_inner(
K.dot(x, self.W_inner[depth, :, :]) + self.b_inner[depth, 0, :]),
sequences=new_preactivated_attributes[:-1, :].T
) # (N_atom x N_features -1)
# Append last feature (bond flag) after the loop
new_attributes = K.concatenate((new_attributes, attributes[:, -1:]),
axis=1)
new_attributes.name = 'new_attributes'
############ UPDATE GRAPH TENSOR WITH NEW ATOM ATTRIBUTES ###################
### Node attribute contribution is located in every entry of graph[i,j,:] where
### there is a bond @ ij or when i = j (self)
# Get atoms matrix (identity)
atoms = T.identity_like(bonds) # (N_atom x N_atom)
atoms.name = 'atoms_identity'
# Combine
bonds_or_atoms = bonds + atoms # (N_atom x N_atom)
bonds_or_atoms.name = 'bonds_or_atoms'
atom_indeces = T.arange(
ones_vec.shape[0]) # 0 to N_atoms - 1 (indeces)
atom_indeces.name = 'atom_indeces vector'
### Subtract previous node attribute contribution
# Multiply each entry in bonds_or_atoms by the previous atom features for that column
(old_features_to_sub, updates) = theano.scan(
lambda i: T.outer(bonds_or_atoms[:, i], attributes[i, :]),
sequences=T.arange(ones_vec.shape[0]))
old_features_to_sub.name = 'old_features_to_sub'
### Add new node attribute contribution
# Multiply each entry in bonds_or_atoms by the previous atom features for that column
(new_features_to_add, updates) = theano.scan(
lambda i: T.outer(bonds_or_atoms[:, i], new_attributes[i, :]),
sequences=T.arange(ones_vec.shape[0]))
new_features_to_add.name = 'new_features_to_add'
# Update new graph
new_graph = graph - old_features_to_sub + new_features_to_add
new_graph.name = 'new_graph'
return (new_attributes, new_graph)
def build(self, img_shape):
if self.model is not None:
print("PosNet has been constructed")
else:
img_input = Input(shape=(img_shape[0], img_shape[1], img_shape[2]),
name='inputImg')
x_conv = Conv2D(24, (8, 8),
padding="valid",
strides=(2, 2),
name="conv1")(img_input)
x_conv = BatchNormalization()(x_conv)
x_conv = Activation('elu')(x_conv)
print(x_conv.get_shape())
x_conv = Conv2D(36, (5, 5),
padding="valid",
strides=(2, 2),
name="conv2")(x_conv)
x_conv = BatchNormalization()(x_conv)
x_conv = Activation('elu')(x_conv)
print(x_conv.get_shape())
x_conv = Conv2D(48, (5, 5),
padding="valid",
strides=(2, 2),
name="conv3")(x_conv)
x_conv = BatchNormalization()(x_conv)
x_conv = Activation('elu')(x_conv)
print(x_conv.get_shape())
x_conv = Conv2D(64, (5, 5), padding="valid", name="conv4")(x_conv)
x_conv = BatchNormalization()(x_conv)
x_conv = Activation('elu')(x_conv)
print(x_conv.get_shape())
x_conv = Conv2D(64, (5, 5), padding="valid", name="conv5")(x_conv)
x_conv = BatchNormalization()(x_conv)
x_conv = Activation('elu')(x_conv)
print(x_conv.get_shape())
x_out = Flatten()(x_conv)
print(x_out.get_shape())
# Cut for transfer learning is here:
speed_input = Input(shape=(1, ), name='inputSpeed')
x_out = Lambda(lambda x: K.concatenate(x, axis=1))(
[x_out, speed_input])
x_out = Dense(200)(x_out)
x_out = BatchNormalization()(x_out)
x_out = Activation('elu')(x_out)
x_out = Dense(200)(x_out)
x_out = BatchNormalization()(x_out)
x_end = Activation('elu')(x_out)
# Branching from X_END to three branches (steer, throttle, position)
steer = Dense(100)(x_end)
steer = BatchNormalization()(steer)
steer = Activation('elu')(steer)
steer = Dropout(.2)(steer)
steer = Dense(30)(steer)
steer = BatchNormalization()(steer)
steer = Activation('elu')(steer)
steer = Dense(1, activation='sigmoid')(steer)
steer = Lambda(lambda x: x * 10 - 5, name='outputSteer')(steer)
throttle = Dense(100, name='thr1')(x_end)
throttle = BatchNormalization(name='thr2')(throttle)
throttle = Activation('elu')(throttle)
throttle = Dropout(.2)(throttle)
throttle = Dense(30, name='thr3')(throttle)
throttle = BatchNormalization(name='thr4')(throttle)
throttle = Activation('elu')(throttle)
throttle = Dense(1, activation='sigmoid', name='thr5')(throttle)
throttle = Lambda(lambda x: x * 2 - 1, name='outputThr')(throttle)
position = Dropout(.3)(x_end)
position = Dense(1, activation='sigmoid', name='pos5')(position)
position = Lambda(lambda x: x * 2 - 1, name='outputPos')(position)
self.model = Model((img_input, speed_input),
(steer, throttle, position))
def call(self, inputs):
channel_axis = 1 if self.data_format == 'channels_first' else -1
input_dim = K.shape(inputs)[channel_axis] // 4
index2 = self.filters * 2
index3 = self.filters * 3
if self.rank == 1:
f_r = self.kernel[:, :, :self.filters]
f_i = self.kernel[:, :, self.filters:index2]
f_j = self.kernel[:, :, index2:index3]
f_k = self.kernel[:, :, index3:]
elif self.rank == 2:
f_r = self.kernel[:, :, :, :self.filters]
f_i = self.kernel[:, :, :, self.filters:index2]
f_j = self.kernel[:, :, :, index2:index3]
f_k = self.kernel[:, :, :, index3:]
elif self.rank == 3:
f_r = self.kernel[:, :, :, :, :self.filters]
f_i = self.kernel[:, :, :, :, self.filters:index2]
f_j = self.kernel[:, :, :, :, index2:index3]
f_k = self.kernel[:, :, :, :, index3:]
convArgs = {
"strides":
self.strides[0] if self.rank == 1 else self.strides,
"padding":
self.padding,
"data_format":
self.data_format,
"dilation_rate":
self.dilation_rate[0] if self.rank == 1 else self.dilation_rate
}
convFunc = {1: K.conv1d, 2: K.conv2d, 3: K.conv3d}[self.rank]
#
# Performing quaternion convolution
#
f_r._keras_shape = self.kernel_shape
f_i._keras_shape = self.kernel_shape
f_j._keras_shape = self.kernel_shape
f_k._keras_shape = self.kernel_shape
cat_kernels_4_r = K.concatenate([f_r, -f_i, -f_j, -f_k], axis=-2)
cat_kernels_4_i = K.concatenate([f_i, f_r, -f_k, f_j], axis=-2)
cat_kernels_4_j = K.concatenate([f_j, f_k, f_r, -f_i], axis=-2)
cat_kernels_4_k = K.concatenate([f_k, -f_j, f_i, f_r], axis=-2)
cat_kernels_4_quaternion = K.concatenate([
cat_kernels_4_r, cat_kernels_4_i, cat_kernels_4_j, cat_kernels_4_k
],
axis=-1)
cat_kernels_4_quaternion._keras_shape = self.kernel_size + (
4 * input_dim, 4 * self.filters)
output = convFunc(inputs, cat_kernels_4_quaternion, **convArgs)
if self.use_bias:
output = K.bias_add(output,
self.bias,
data_format=self.data_format)
if self.activation is not None:
output = self.activation(output)
return output
def conductance_model(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=False):
reg_val = 1
v_leak = 0
v_exc = 60
v_inh = -30
g_leak = 1
filter_shape[1] = 1
pad_x = int((filter_shape[1] - 1))+2
pad_t = int((filter_shape[0] - 1))
# Define the input as a tensor with shape input_shape
image_in = Input(input_shape)
s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in)
s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in)
s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in)
g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s1)
g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s2)
g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s3)
if fit_reversal:
expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1))
squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1))
g1 = expand_last(g1)
g2 = expand_last(g2)
g3 = expand_last(g3)
numerator_in = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1, g2, g3])
numerator = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)(numerator_in)
denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3])
vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator])
vm = squeeze_last(vm)
else:
g1_v_inh = Lambda(lambda lam: lam * v_inh)(g1)
g2_v_exc = Lambda(lambda lam: lam * v_exc)(g2)
g3_v_inh = Lambda(lambda lam: lam * v_inh)(g3)
numerator = Lambda(lambda inputs: inputs[0] + inputs[1] + inputs[2])([g1_v_inh, g2_v_exc, g3_v_inh])
denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3])
vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator])
vm_bias = BiasLayer()(vm)
vm_rect = Lambda(lambda lam: K.relu(lam))(vm_bias)
if sum_over_space:
conv_x_size = vm.get_shape().as_list()[2]
else:
conv_x_size = 1
combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)(vm_rect)
# Create model
model = Model(inputs=image_in, outputs=combine_filters, name='conductance_model')
return model, pad_x, pad_t
def max_min_mean_operator(x5d):
max = K.max(x5d, axis=1, keepdims=True)
min = K.min(x5d, axis=1, keepdims=True)
mean = K.mean(x5d, axis=1, keepdims=True)
return K.concatenate([max, min, mean], axis=1)
def shift_right(x, offset=1):
assert offset > 0
return K.concatenate([K.zeros_like(x[:, :offset]), x[:, :-offset]], axis=1)
def step(self, a, states): # 重载rnn中的step
r_tm1 = states[:self.nb_layers] # 读取输入的R、E、C（上时刻状态）
c_tm1 = states[self.nb_layers:2*self.nb_layers]
e_tm1 = states[2*self.nb_layers:3*self.nb_layers]
if self.extrap_start_time is not None:
t = states[-1]
a = K.switch(t >= self.t_extrap, states[-2], a) # if past self.extrap_start_time, the previous prediction will be treated as the actual
c = []
r = []
e = []
# R Unit
for l in reversed(range(self.nb_layers)): # 由于R的计算需要前时刻和高一层的R，因此需要由上向下进行计算
inputs = [r_tm1[l], e_tm1[l]]
if l < self.nb_layers - 1:
inputs.append(r_up) # 除了最高层，前面的输入都是R_t-1,R_l+1,E,以及隐含的状态C
# 标准LSTM过程
inputs = K.concatenate(inputs, axis=self.channel_axis) # 把各个特征图放到一起
i = self.conv_layers['i'][l].call(inputs) # 按照相应的卷积门尺寸卷积
f = self.conv_layers['f'][l].call(inputs)
o = self.conv_layers['o'][l].call(inputs)
_c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs) # c_t = f*c_t-1 + i*tanh(inputs)
_r = o * self.LSTM_activation(_c) # r_t = o*tanh(c_t)
c.insert(0, _c)
r.insert(0, _r)
if l > 0:
r_up = self.upsample.call(_r) # 上采样
for l in range(self.nb_layers):
ahat = self.conv_layers['ahat'][l].call(r[l]) # Ahat是R的卷积
if l == 0:
ahat = K.minimum(ahat, self.pixel_max) # 第一层，Ahat限幅，准备作为输出图像
frame_prediction = ahat # 当output_mode == 'prediction'时输出
# compute errors
e_up = self.error_activation(ahat - a)
e_down = self.error_activation(a - ahat)
e.append(K.concatenate((e_up, e_down), axis=self.channel_axis))
if self.output_layer_num == l:
if self.output_layer_type == 'A':
output = a
elif self.output_layer_type == 'Ahat':
output = ahat
elif self.output_layer_type == 'R':
output = r[l]
elif self.output_layer_type == 'E':
output = e[l]
if l < self.nb_layers - 1:
a = self.conv_layers['a'][l].call(e[l])
a = self.pool.call(a) # target for next layer
if self.output_layer_type is None:
if self.output_mode == 'prediction':
output = frame_prediction
else:
for l in range(self.nb_layers):
layer_error = K.mean(K.batch_flatten(e[l]), axis=-1, keepdims=True) # 各层平均误差，每层一个数
all_error = layer_error if l == 0 else K.concatenate((all_error, layer_error), axis=-1)
if self.output_mode == 'error':
output = all_error
else:
output = K.concatenate((K.batch_flatten(frame_prediction), all_error), axis=-1)
states = r + c + e
if self.extrap_start_time is not None:
states += [frame_prediction, t + 1]
return output, states
def call(self, inputs):
inputs -= K.mean(inputs, axis=1, keepdims=True)
inputs = K.l2_normalize(inputs, axis=1)
pos = K.relu(inputs)
neg = K.relu((- inputs))
return K.concatenate([pos, neg], axis=1)
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [0, 1, 2]]
input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 8, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(lambda b, *args: b < m,
loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
(1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss
if print_loss:
loss = tf.Print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
K.sum(ignore_mask)
],
message='loss: ')
return loss
def call(self, inputs):
if isinstance(inputs, list):
x, x_mask = inputs
else:
x, x_mask = inputs, None
seq_dim = K.int_shape(x)[-1]
# 补足长度，保证可以reshape
seq_len = K.shape(x)[1]
pad_len = self.rate - seq_len % self.rate
x = K.temporal_padding(x, (0, pad_len))
if x_mask is not None:
x_mask = K.temporal_padding(x_mask, (0, pad_len))
new_seq_len = K.shape(x)[1]
x = K.reshape(x, (-1, new_seq_len,
seq_dim)) # 经过padding后shape可能变为None，所以重新声明一下shape
# 线性变换
qw = self.reuse(self.q_dense, x)
kw = self.reuse(self.k_dense, x)
vw = self.reuse(self.v_dense, x)
# 提取局部特征
kernel_size = 1 + 2 * self.neighbors
kwp = extract_seq_patches(
kw, kernel_size,
self.rate) # shape=[None, seq_len, kernel_size, out_dim]
vwp = extract_seq_patches(
vw, kernel_size,
self.rate) # shape=[None, seq_len, kernel_size, out_dim]
if x_mask is not None:
xp_mask = extract_seq_patches(x_mask, kernel_size, self.rate)
# 形状变换
qw = K.reshape(qw, (-1, new_seq_len // self.rate, self.rate,
self.heads, self.key_size))
kw = K.reshape(kw, (-1, new_seq_len // self.rate, self.rate,
self.heads, self.key_size))
vw = K.reshape(vw, (-1, new_seq_len // self.rate, self.rate,
self.heads, self.size_per_head))
kwp = K.reshape(kwp, (-1, new_seq_len // self.rate, self.rate,
kernel_size, self.heads, self.key_size))
vwp = K.reshape(vwp, (-1, new_seq_len // self.rate, self.rate,
kernel_size, self.heads, self.size_per_head))
if x_mask is not None:
x_mask = K.reshape(x_mask,
(-1, new_seq_len // self.rate, self.rate, 1, 1))
xp_mask = K.reshape(
xp_mask,
(-1, new_seq_len // self.rate, self.rate, kernel_size, 1, 1))
# 维度置换
qw = K.permute_dimensions(
qw, (0, 3, 2, 1, 4)) # shape=[None, heads, r, seq_len // r, size]
kw = K.permute_dimensions(kw, (0, 3, 2, 1, 4))
vw = K.permute_dimensions(vw, (0, 3, 2, 1, 4))
qwp = K.expand_dims(qw, 4)
kwp = K.permute_dimensions(
kwp,
(0, 4, 2, 1, 3,
5)) # shape=[None, heads, r, seq_len // r, kernel_size, out_dim]
vwp = K.permute_dimensions(vwp, (0, 4, 2, 1, 3, 5))
if x_mask is not None:
x_mask = K.permute_dimensions(x_mask, (0, 3, 2, 1, 4))
xp_mask = K.permute_dimensions(xp_mask, (0, 4, 2, 1, 3, 5))
# Attention1
a = K.batch_dot(qw, kw, [4, 4]) / self.key_size**0.5
a = K.permute_dimensions(a, (0, 1, 2, 4, 3))
a = to_mask(a, x_mask, 'add')
a = K.permute_dimensions(a, (0, 1, 2, 4, 3))
if self.mask_right:
ones = K.ones_like(a[:1, :1, :1])
mask = (ones - K.tf.matrix_band_part(ones, -1, 0)) * 1e10
a = a - mask
# Attention2
ap = K.batch_dot(qwp, kwp, [5, 5]) / self.key_size**0.5
ap = K.permute_dimensions(ap, (0, 1, 2, 3, 5, 4))
if x_mask is not None:
ap = to_mask(ap, xp_mask, 'add')
ap = K.permute_dimensions(ap, (0, 1, 2, 3, 5, 4))
if self.mask_right:
mask = np.ones((1, kernel_size))
mask[:, -self.neighbors:] = 0
mask = (1 - K.constant(mask)) * 1e10
for _ in range(4):
mask = K.expand_dims(mask, 0)
ap = ap - mask
ap = ap[..., 0, :]
# 合并两个Attention
A = K.concatenate([a, ap], -1)
A = K.softmax(A)
a, ap = A[..., :K.shape(a)[-1]], A[..., K.shape(a)[-1]:]
# 完成输出1
o1 = K.batch_dot(a, vw, [4, 3])
# 完成输出2
ap = K.expand_dims(ap, -2)
o2 = K.batch_dot(ap, vwp, [5, 4])
o2 = o2[..., 0, :]
# 完成输出
o = o1 + o2
o = to_mask(o, x_mask, 'mul')
o = K.permute_dimensions(o, (0, 3, 2, 1, 4))
o = K.reshape(o, (-1, new_seq_len, self.out_dim))
o = o[:, :-pad_len]
return o
def call(self, inputs, mask=None):
# premise_length = hypothesis_length in the following lines, but the names are kept separate to keep
# track of the axes being normalized.
# The inputs can be a two different tensors, or a concatenation. Hence, the conditional below.
if isinstance(inputs, list) or isinstance(inputs, tuple):
premise_embedding, hypothesis_embedding = inputs
# (batch_size, premise_length), (batch_size, hypothesis_length)
premise_mask, hypothesis_mask = mask
else:
premise_embedding = inputs[:, :self.premise_length, :]
hypothesis_embedding = inputs[:, self.premise_length:, :]
# (batch_size, premise_length), (batch_size, hypothesis_length)
premise_mask = None if mask is None else mask[:, :self.
premise_length]
hypothesis_mask = None if mask is None else mask[:, self.
premise_length:]
if premise_mask is not None:
premise_embedding = switch(K.expand_dims(premise_mask),
premise_embedding,
K.zeros_like(premise_embedding))
if hypothesis_mask is not None:
hypothesis_embedding = switch(K.expand_dims(hypothesis_mask),
hypothesis_embedding,
K.zeros_like(hypothesis_embedding))
activation = activations.get(self.hidden_layer_activation)
# (batch_size, premise_length, hidden_dim)
projected_premise = apply_feed_forward(premise_embedding,
self.attend_weights, activation)
# (batch_size, hypothesis_length, hidden_dim)
projected_hypothesis = apply_feed_forward(hypothesis_embedding,
self.attend_weights,
activation)
## Step 1: Attend
p2h_alignment = self._align(projected_premise, projected_hypothesis,
premise_mask, hypothesis_mask)
# beta in the paper (equation 2)
# (batch_size, premise_length, emb_dim)
p2h_attention = self._attend(hypothesis_embedding, p2h_alignment,
self.premise_length)
h2p_alignment = self._align(projected_hypothesis, projected_premise,
hypothesis_mask, premise_mask)
# alpha in the paper (equation 2)
# (batch_size, hyp_length, emb_dim)
h2p_attention = self._attend(premise_embedding, h2p_alignment,
self.hypothesis_length)
## Step 2: Compare
# Equation 3 in the paper.
compared_premise = self._compare(premise_embedding, p2h_attention)
compared_hypothesis = self._compare(hypothesis_embedding,
h2p_attention)
## Step 3: Aggregate
# Equations 4 and 5.
# (batch_size, hidden_dim * 2)
aggregated_input = K.concatenate([
K.sum(compared_premise, axis=1),
K.sum(compared_hypothesis, axis=1)
])
# (batch_size, hidden_dim)
input_to_scorer = apply_feed_forward(aggregated_input,
self.aggregate_weights,
activation)
# (batch_size, 2)
final_activation = activations.get(self.final_activation)
scores = final_activation(K.dot(input_to_scorer, self.scorer))
return scores
def call(self, inputs, training=None):
# inputs.shape=[None, input_num_capsule, input_dim_capsule]
# inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule]
inputs_expand = K.expand_dims(inputs, 1)
# Replicate num_capsule dimension to prepare being multiplied by W
# inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule]
inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1])
# Compute `inputs * W` by scanning inputs_tiled on dimension 0.
# x.shape=[num_capsule, input_num_capsule, input_dim_capsule]
# W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule]
# Regard the first two dimensions as `batch` dimension,
# then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule].
# inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule]
inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]),
elems=inputs_tiled)
"""
# Begin: routing algorithm V1, dynamic ------------------------------------------------------------#
# The prior for coupling coefficient, initialized as zeros.
b = K.zeros(shape=[self.batch_size, self.num_capsule, self.input_num_capsule])
def body(i, b, outputs):
c = tf.nn.softmax(b, dim=1) # dim=2 is the num_capsule dimension
outputs = squash(K.batch_dot(c, inputs_hat, [2, 2]))
if i != 1:
b = b + K.batch_dot(outputs, inputs_hat, [2, 3])
return [i-1, b, outputs]
cond = lambda i, b, inputs_hat: i > 0
loop_vars = [K.constant(self.num_routing), b, K.sum(inputs_hat, 2, keepdims=False)]
shape_invariants = [tf.TensorShape([]),
tf.TensorShape([None, self.num_capsule, self.input_num_capsule]),
tf.TensorShape([None, self.num_capsule, self.dim_capsule])]
_, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants)
# End: routing algorithm V1, dynamic ------------------------------------------------------------#
"""
# Begin: Routing algorithm ---------------------------------------------------------------------#
# In forward pass, `inputs_hat_stopped` = `inputs_hat`;
# In backward, no gradient can flow from `inputs_hat_stopped` back to `inputs_hat`.
inputs_hat_stopped = K.stop_gradient(inputs_hat)
# The prior for coupling coefficient, initialized as zeros.
# b.shape = [None, self.num_capsule, self.input_num_capsule].
b = tf.zeros(shape=[
K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule
])
#assert self.num_routing > 0, 'The num_routing should be > 0.'
for i in range(self.num_routing):
# c.shape=[batch_size, num_capsule, input_num_capsule]
c = tf.nn.softmax(b, dim=1)
# At last iteration, use `inputs_hat` to compute `outputs` in order to backpropagate gradient
if i == self.num_routing - 1:
# c.shape = [batch_size, num_capsule, input_num_capsule]
# inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
# The first two dimensions as `batch` dimension,
# then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule].
# outputs.shape=[None, num_capsule, dim_capsule]
outputs = squash(K.batch_dot(c, inputs_hat,
[2, 2])) # [None, 10, 16]
else: # Otherwise, use `inputs_hat_stopped` to update `b`. No gradients flow on this path.
outputs = squash(K.batch_dot(c, inputs_hat_stopped, [2, 2]))
# outputs.shape = [None, num_capsule, dim_capsule]
# inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
# The first two dimensions as `batch` dimension,
# then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule].
# b.shape=[batch_size, num_capsule, input_num_capsule]
b += K.batch_dot(outputs, inputs_hat_stopped, [2, 3])
# End: Routing algorithm -----------------------------------------------------------------------#
all = K.concatenate([outputs, c])
return all
def main():
width, height = load_img(target_image_path).size
global img_height, img_width
img_height = 200
img_width = int(width * img_height / height)
target_image = K.constant(preprocess_image(target_image_path))
style_reference_image = K.constant(preprocess_image(style_reference_path))
combination_image = K.placeholder((1, img_height, img_width, 3))
input_tensor = K.concatenate(
[target_image, style_reference_image, combination_image], axis=0)
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
output_dict = dict([(layer.name, layer.output) for layer in model.layers])
content_layer = 'block5_conv2'
style_layers = [
'block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1',
'block5_conv1'
]
total_variation_weight = 1e-4
style_weight = 1.
content_weight = 0.025
loss = K.variable(0.)
layer_features = output_dict[content_layer]
target_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss = loss + content_weight * content_loss(target_image_features,
combination_features)
for layer_name in style_layers:
layer_features = output_dict[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
s1 = style_loss(style_reference_features, combination_features)
loss = loss + (style_weight / len(style_layers)) * s1
loss = loss + total_variation_weight * total_variation_loss(
combination_image)
grads = K.gradients(loss, combination_image)[0]
fetch_loss_and_grads = K.function([combination_image], [loss, grads])
class Evaluator(object):
def __init__(self):
self.loss_value = None
self.grads_values = None
def loss(self, x):
assert self.loss_value is None
x = x.reshape((1, img_height, img_width, 3))
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1].flatten().astype('float64')
self.loss_value = loss_value
self.grad_values = grad_values
return self.loss_value
def grads(self, x):
assert self.loss_value is not None
grad_values = np.copy(self.grad_values)
self.loss_value = None
self.grad_values = None
return grad_values
evaluator = Evaluator()
iterations = iter_size
x = preprocess_image(target_image_path)
x = x.flatten()
for i in range(iterations):
x, min_val, info = fmin_l_bfgs_b(evaluator.loss,
x,
fprime=evaluator.grads,
maxfun=20)
img = x.copy().reshape((img_height, img_width, 3))
img = deprocess_image(img)
percent = (int)(100.0 * i / iterations)
sys.stdout.write("\r{0}{1}{2}{3}{4}".format(
"\r[%2d%%]" % percent, "[", "=" * int(percent / 5),
" " * (20 - int(percent / 5)), "]"))
sys.stdout.flush()
if (i == iterations - 1):
fname = 'NST.png'
save_img(fname, img)
print('\n\rImage saved as %s \n\r' % fname)
print('\n\r')
return image_array[:, :, :, ::-1]
conv_style = convert_to_bgr(style_array)
conv_content = convert_to_bgr(content_array)
# Feeding images as variable into Keras
style_var = backend.variable(conv_style)
content_var = backend.variable(conv_content)
combination_img = backend.placeholder((1, h, w, 3))
# Since using Tensorflow backend and it needs tensors, we convert the image data into one concatened tensor.
input_tensor = backend.concatenate([content_var, style_var, combination_img],
axis=0)
# Getting the model ready
model = VGG16(input_tensor=input_tensor, weights="imagenet", include_top=False)
# Let's load the layers
layers = dict([(layer.name, layer.output) for layer in model.layers])
layers
# arbitrary values
content_weight = 0.025
style_weight = 5.0
total_variation_weight = 1.0
