def step(self, x, states): h = states[0] # states[1] necessary? # comes from the constants X_static = states[-2] # equals K.dot(static_x, self._W1) + self._b2 with X.shape=[bs, L, static_input_dim] total_x_static_prod = states[-1] # expand dims to add the vector which is only valid for this time step # to total_x_prod which is valid for all time steps hw = K.expand_dims(K.dot(h, self._W2), 1) additive_atn = total_x_static_prod + hw attention = K.softmax(K.dot(additive_atn, self._V), axis=1) static_x_weighted = K.sum(attention * X_static, [1]) x = K.dot(K.concatenate([x, static_x_weighted], 1), self._W3) + self._b3 h, new_states = self.layer.cell.call(x, states[:-2]) # append attention to the states to "smuggle" it out of the RNN wrapper attention = K.squeeze(attention, -1) h = K.concatenate([h, attention]) return h, new_states
def inner_loss(y_true, y_pred): # Workaround until https://github.com/plaidml/plaidml/pull/284 is accepted if K.backend() == "plaidml.keras.backend": y_true = K.reshape(y_true, y_pred.shape.dims) n_true = K.concatenate([y_true[:, :, :, i:i+1] * mask for i in range(3)], axis=-1) n_pred = K.concatenate([y_pred[:, :, :, i:i+1] * mask for i in range(3)], axis=-1) return loss_func(n_true, n_pred)
def correct_boxes(box_xy, box_wh, input_shape, image_shape): '''Get corrected boxes''' box_yx = box_xy[..., ::-1] box_hw = box_wh[..., ::-1] input_shape = K.cast(input_shape, K.dtype(box_yx)) image_shape = K.cast(image_shape, K.dtype(box_yx)) new_shape = K.round(image_shape * K.min(input_shape / image_shape)) offset = (input_shape - new_shape) / 2. / input_shape scale = input_shape / new_shape box_yx = (box_yx - offset) * scale box_hw *= scale box_mins = box_yx - (box_hw / 2.) box_maxes = box_yx + (box_hw / 2.) boxes = K.concatenate([ box_mins[..., 0:1], # y_min box_mins[..., 1:2], # x_min box_maxes[..., 0:1], # y_max box_maxes[..., 1:2] # x_max ]) # Scale boxes back to original image shape. boxes *= K.concatenate([image_shape, image_shape]) return boxes
def batched_dot(input, W_list, b_list, sizes_list): X = input W = K.concatenate(W_list, axis=-1) b = K.concatenate(b_list, axis=-1) Y = K.dot(X, W) + b sl = [0, ] + list(np.cumsum(sizes_list)) return [Y[:, sl[i]:sl[i+1]] for i in range(len(sizes_list))]
def test_sparse_concat(self): x_d = np.array([0, 7, 2, 3], dtype=np.float32) x_r = np.array([0, 2, 2, 3], dtype=np.int64) x_c = np.array([4, 3, 2, 3], dtype=np.int64) x_sparse_1 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5)) x_d = np.array([0, 7, 2, 3], dtype=np.float32) x_r = np.array([0, 2, 2, 3], dtype=np.int64) x_c = np.array([4, 3, 2, 3], dtype=np.int64) x_sparse_2 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5)) x_dense_1 = x_sparse_1.toarray() x_dense_2 = x_sparse_2.toarray() backends = [KTF] if KTH.th_sparse_module: # Theano has some dependency issues for sparse backends.append(KTH) for K in backends: k_s = K.concatenate([K.variable(x_sparse_1), K.variable(x_sparse_2)]) assert K.is_sparse(k_s) k_s_d = K.eval(k_s) k_d = K.eval(K.concatenate([K.variable(x_dense_1), K.variable(x_dense_2)])) assert k_s_d.shape == k_d.shape assert_allclose(k_s_d, k_d, atol=1e-05)
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5): '''Return yolo_loss tensor Parameters ---------- yolo_outputs: list of tensor, the output of yolo_body y_true: list of array, the output of preprocess_true_boxes anchors: array, shape=(T, 2), wh num_classes: integer ignore_thresh: float, the iou threshold whether to ignore object confidence loss Returns ------- loss: tensor, shape=(1,) ''' yolo_outputs = args[:3] y_true = args[3:] anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0])) grid_shapes = [K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(3)] loss = 0 m = K.shape(yolo_outputs[0])[0] for l in range(3): object_mask = y_true[l][..., 4:5] true_class_probs = y_true[l][..., 5:] pred_xy, pred_wh, pred_confidence, pred_class_probs = yolo_head(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape) pred_box = K.concatenate([pred_xy, pred_wh]) # Darknet box loss. xy_delta = (y_true[l][..., :2]-pred_xy)*grid_shapes[l][::-1] wh_delta = K.log(y_true[l][..., 2:4]) - K.log(pred_wh) # Avoid log(0)=-inf. wh_delta = K.switch(object_mask, wh_delta, K.zeros_like(wh_delta)) box_delta = K.concatenate([xy_delta, wh_delta], axis=-1) box_delta_scale = 2 - y_true[l][...,2:3]*y_true[l][...,3:4] # Find ignore mask, iterate over each of batch. ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True) object_mask_bool = K.cast(object_mask, 'bool') def loop_body(b, ignore_mask): true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0]) iou = box_iou(pred_box[b], true_box) best_iou = K.max(iou, axis=-1) ignore_mask = ignore_mask.write(b, K.cast(best_iou<ignore_thresh, K.dtype(true_box))) return b+1, ignore_mask _, ignore_mask = K.control_flow_ops.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask]) ignore_mask = ignore_mask.stack() ignore_mask = K.expand_dims(ignore_mask, -1) box_loss = object_mask * K.square(box_delta*box_delta_scale) confidence_loss = object_mask * K.square(1-pred_confidence) + \ (1-object_mask) * K.square(0-pred_confidence) * ignore_mask class_loss = object_mask * K.square(true_class_probs-pred_class_probs) loss += K.sum(box_loss) + K.sum(confidence_loss) + K.sum(class_loss) return loss / K.cast(m, K.dtype(loss))
def call(self, argument, mask=None): """Execute this layer on input tensors. Parameters ---------- argument: list List of two tensors (X, Xp). X should be of shape (n_test, n_feat) and Xp should be of shape (n_support, n_feat) where n_test is the size of the test set, n_support that of the support set, and n_feat is the number of per-atom features. Returns ------- list Returns two tensors of same shape as input. Namely the output shape will be [(n_test, n_feat), (n_support, n_feat)] """ x, xp = argument # Get initializations p = self.p_init q = self.q_init # Rename support z = xp states = self.support_states_init x_states = self.test_states_init for d in range(self.max_depth): # Process support xp using attention e = cos(z + q, xp) a = K.softmax(e) # Get linear combination of support set r = K.dot(a, xp) # Not sure if it helps to place the update here or later yet. Will # decide # z = r # Process test x using attention x_e = cos(x + p, z) x_a = K.softmax(x_e) s = K.dot(x_a, z) # Generate new support attention states qr = K.concatenate([q, r], axis=1) q, states = self.support_lstm([qr] + states) # Generate new test attention states ps = K.concatenate([p, s], axis=1) p, x_states = self.test_lstm([ps] + x_states) # Redefine z = r # return [x+p, z+q] return [x + p, xp + q]
def call(self, x, mask=None): stride_row, stride_col = self.subsample _, feature_dim, nb_filter = self.W_shape if self.dim_ordering == 'th': if K._backend == 'theano': output = [] for i in range(self.output_row): for j in range(self.output_col): slice_row = slice(i * stride_row, i * stride_row + self.nb_row) slice_col = slice(j * stride_col, j * stride_col + self.nb_col) x_flatten = K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim)) output.append(K.dot(x_flatten, self.W[i * self.output_col + j, :, :])) output = K.concatenate(output, axis=0) else: xs = [] for i in range(self.output_row): for j in range(self.output_col): slice_row = slice(i * stride_row, i * stride_row + self.nb_row) slice_col = slice(j * stride_col, j * stride_col + self.nb_col) xs.append(K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim))) x_aggregate = K.concatenate(xs, axis=0) output = K.batch_dot(x_aggregate, self.W) output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter)) output = K.permute_dimensions(output, (2, 3, 0, 1)) elif self.dim_ordering == 'tf': xs = [] for i in range(self.output_row): for j in range(self.output_col): slice_row = slice(i * stride_row, i * stride_row + self.nb_row) slice_col = slice(j * stride_col, j * stride_col + self.nb_col) xs.append(K.reshape(x[:, slice_row, slice_col, :], (1, -1, feature_dim))) x_aggregate = K.concatenate(xs, axis=0) output = K.batch_dot(x_aggregate, self.W) output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter)) output = K.permute_dimensions(output, (2, 0, 1, 3)) else: raise Exception('Invalid dim_ordering: ' + self.dim_ordering) if self.bias: if self.dim_ordering == 'th': output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col)) elif self.dim_ordering == 'tf': output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter)) else: raise Exception('Invalid dim_ordering: ' + self.dim_ordering) output = self.activation(output) return output
def fun(estimation, next_frame, t1, t2, t3): inputs = concatenate([estimation, next_frame]) A01 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(inputs) C11 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=2)(A01) C12 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C11) C13 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C12) C14 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C13) C14 = add([C11, C14]) C21 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C14) C22 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C21) C23 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C22) C24 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C23) C24 = add([C21, C24]) C31 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=2)(C24) C32 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C31) C33 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C32) C34 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C33) C34 = add([C31, C34]) C41 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=2)(C34) C42 = Lambda(lambda x: K.concatenate([x,t1],axis=-1), output_shape=(nx/8,ny/8,512))(C41) B42 = Conv2D(256, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C42) C43 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(B42) C44 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C43) C45 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C44) C45 = add([C41, C45]) C51 = transposed_convolution(128, kernel_size=4, l2_reg=l2_reg, strides=2)(C45) C51 = add([C51, C34]) C52 = Lambda(lambda x: K.concatenate([x,t2],axis=-1),output_shape=(nx/4,ny/4,256))(C51) B52 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C52) C53 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(B52) C54 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C53) C55 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C54) C55 = add([C51, C55]) C61 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C55) C61 = add([C61, C24]) C62 = Lambda(lambda x: K.concatenate([x,t3],axis=-1),output_shape=(nx/2,ny/2,128))(C61) B62 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C62) C63 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(B62) C64 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C63) C65 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C64) C65 = add([C61, C65]) C71 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C65) C72 = convolution(64, kernel_size=4, l2_reg=l2_reg, strides=1)(C71) out = convolution(1, kernel_size=3, l2_reg=l2_reg, strides=1)(C72) return out, C43, C53, C63
def step(self, x, states): h_tild_tm1 = states[0] B_U = states[1] B_W = states[2] if self.consume_less == 'cpu': x_i = x[:, :self.output_dim] x_f = x[:, self.output_dim: 2 * self.output_dim] x_c = x[:, 2 * self.output_dim: 3 * self.output_dim] x_o = x[:, 3 * self.output_dim: 4 * self.output_dim] x_new = x[:, 4 * self.output_dim:] else: x_i = K.dot(x * B_W[0], self.W_i) + self.b_i x_f = K.dot(x * B_W[1], self.W_f) + self.b_f x_c = K.dot(x * B_W[2], self.W_c) + self.b_c x_o = K.dot(x * B_W[3], self.W_o) + self.b_o x_new = x # self.C_tape -> BT, t-1, k # self.H_tape -> BT, t-1, k # x -> BT, k # h_tild_tm1 -> BT, k if self.H_tape is None: self.H_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1)) self.C_tape = K.zeros_like(h_tild_tm1).dimshuffle((0,'x',1)) # s_t -> BT, t-1, 1 t = K.shape(self.C_tape)[1] sum1 = K.dot(self.H_tape, self.W_h) sum2 = K.dot(K.repeat_elements(x_new.dimshuffle((0,'x',1)),t, axis=1), self.W_x) sum3 = K.dot(K.repeat_elements(h_tild_tm1.dimshuffle((0,'x',1)),t, axis=1), self.W_h_tilde) tanhed_sum = K.tanh(sum1 + sum2 + sum3) a_t = K.dot(tanhed_sum, self.v)[:,:,0] s_t = K.softmax(a_t) h_tilde_t = T.batched_dot(self.H_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0] c_tilde_t = T.batched_dot(self.C_tape.dimshuffle((0,2,1)), s_t.dimshuffle((0,1,'x')))[:,:,0] i = self.inner_activation(x_i + K.dot(h_tilde_t * B_U[0], self.U_i)) f = self.inner_activation(x_f + K.dot(h_tilde_t * B_U[1], self.U_f)) c_t = f * c_tilde_t + i * self.activation(x_c + K.dot(h_tilde_t * B_U[2], self.U_c)) o = self.inner_activation(x_o + K.dot(h_tilde_t * B_U[3], self.U_o)) h_t = o * self.activation(c_t) # Add to Tape self.C_tape = K.concatenate([self.C_tape, c_t.dimshuffle((0,'x',1))], axis=1) self.H_tape = K.concatenate([self.H_tape, h_t.dimshuffle((0,'x',1))], axis=1) return h_t, [h_tilde_t]
def call(self, input): forward_lstm = LSTM(self.units, dropout=self.dropout, return_sequences=True, return_state=True) backward_lstm = LSTM(self.units, dropout=self.dropout, return_sequences=True, return_state=True, go_backwards=True) fw_outputs, fw_output, fw_state = forward_lstm(input) bw_outputs, bw_output, b_state = backward_lstm(input) bw_outputs = K.reverse(bw_outputs, 1) # bw_output = bw_state[0] outputs = K.concatenate([fw_outputs, bw_outputs]) last_output = K.concatenate([fw_output, bw_output]) return [outputs, last_output]
def normals_metric(y_true, y_pred): y_true = K.variable(y_true) y_pred = K.variable(y_pred) y_true = K.expand_dims(y_true,0) filter_y = K.variable(np.array([[ 0., -0.5 , 0.], [0., 0., 0.], [0., 0.5, 0.]]).reshape(3, 3, 1, 1)) filter_x = K.variable(np.array([ [0, 0., 0.], [0.5, 0., -0.5], [0., 0., 0.]]).reshape(3, 3, 1, 1)) dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same') dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same') dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx)) dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy)) mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx)) mag_norm = K.sqrt(mag_norm) N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm N1 = dzdx_ / mag_norm N2 = dzdy_ / mag_norm normals = K.concatenate(tensors=[N1,N2,N3],axis=-1) dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same') dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same') mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0 mag_norm_pred = K.sqrt(mag_norm_pred) grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred, 0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1) grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred, 1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1) dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1) dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1) dot_term_x = K.abs(dot_term_x) dot_term_y = K.abs(dot_term_y) return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def call(self, x, mask=None): if (self.size == None) or (self.mode == 'sum'): self.size = int(x.shape[-1]) batch_size, seq_len = K.shape(x)[0], K.shape(x)[1] position_j = 1. / K.pow(10000., 2 * K.arange(self.size / 2, dtype='float32') / self.size) position_j = K.expand_dims(position_j, 0) position_i = K.cumsum(K.ones_like(x[:, :, 0]), 1) - 1 # K.arange不支持变长,只好用这种方法生成 position_i = K.expand_dims(position_i, 2) position_ij = K.dot(position_i, position_j) position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2) if self.mode == 'sum': return position_ij + x elif self.mode == 'concat': return K.concatenate([position_ij, x], 2)
def sparse_amsoftmax_loss(y_true, y_pred, scale=30, margin=0.35): y_true = K.expand_dims(y_true[:, 0], 1) # 保证y_true的shape=(None, 1) y_true = K.cast(y_true, 'int32') # 保证y_true的dtype=int32 batch_idxs = K.arange(0, K.shape(y_true)[0]) batch_idxs = K.expand_dims(batch_idxs, 1) idxs = K.concatenate([batch_idxs, y_true], 1) y_true_pred = K.tf.gather_nd(y_pred, idxs) # 目标特征,用tf.gather_nd提取出来 y_true_pred = K.expand_dims(y_true_pred, 1) y_true_pred_margin = y_true_pred - margin # 减去margin _Z = K.concatenate([y_pred, y_true_pred_margin], 1) # 为计算配分函数 _Z = _Z * scale # 缩放结果,主要因为pred是cos值,范围[-1, 1] logZ = K.logsumexp(_Z, 1, keepdims=True) # 用logsumexp,保证梯度不消失 logZ = logZ + K.log(1 - K.exp(scale * y_true_pred - logZ)) # 从Z中减去exp(scale * y_true_pred) return - y_true_pred_margin * scale + logZ
def call(self, X, mask=None): def reverse(x): if K.ndim(x) == 2: x = K.expand_dims(x, -1) rev = K.permute_dimensions(x, (1, 0, 2))[::-1] rev = K.squeeze(rev, -1) else: rev = K.permute_dimensions(x, (1, 0, 2))[::-1] return K.permute_dimensions(rev, (1, 0, 2)) Y = self.forward.call(X, mask) # 0,0,0,1,3,6,10 X_rev = reverse(X) # 4,3,2,1,0,0,0 mask_rev = reverse(mask) if mask else None # 1,1,1,1,0,0,0 Y_rev = self.reverse.call(X_rev, mask_rev) # 4,7,9,10,10,10,10 #Fix allignment if self.return_sequences: Y_rev = reverse(Y_rev) # 10,10,10,10,9,7,4 if self.merge_mode == 'concat': return K.concatenate([Y, Y_rev]) elif self.merge_mode == 'sum': return Y + Y_rev elif self.merge_mode == 'ave': return (Y + Y_rev) / 2 elif self.merge_mode == 'mul': return Y * Y_rev
def test_lambda(): from keras.utils.layer_utils import layer_from_config Lambda = core.Lambda layer_test(Lambda, kwargs={'function': lambda x: x + 1}, input_shape=(3, 2)) # test serialization with function def f(x): return x + 1 ld = Lambda(f) config = ld.get_config() ld = layer_from_config({'class_name': 'Lambda', 'config': config}) ld = Lambda(lambda x: K.concatenate([K.square(x), x]), output_shape=lambda s: tuple(list(s)[:-1] + [2 * s[-1]])) config = ld.get_config() ld = Lambda.from_config(config) # test serialization with output_shape function def f(x): return K.concatenate([K.square(x), x]) def f_shape(s): return tuple(list(s)[:-1] + [2 * s[-1]]) ld = Lambda(f, output_shape=f_shape) config = ld.get_config() ld = layer_from_config({'class_name': 'Lambda', 'config': config})
def call(self, x, mask=None): input_shape = K.shape(x) if self.dim_ordering == 'th': num_rows = input_shape[2] num_cols = input_shape[3] elif self.dim_ordering == 'tf': num_rows = input_shape[1] num_cols = input_shape[2] row_length = [K.cast(num_rows, 'float32') / i for i in self.pool_list] col_length = [K.cast(num_cols, 'float32') / i for i in self.pool_list] outputs = [] if self.dim_ordering == 'th': for pool_num, num_pool_regions in enumerate(self.pool_list): for ix in range(num_pool_regions): for jy in range(num_pool_regions): x1 = ix * col_length[pool_num] x2 = ix * col_length[pool_num] + col_length[pool_num] y1 = jy * row_length[pool_num] y2 = jy * row_length[pool_num] + row_length[pool_num] x1 = K.cast(K.round(x1), 'int32') x2 = K.cast(K.round(x2), 'int32') y1 = K.cast(K.round(y1), 'int32') y2 = K.cast(K.round(y2), 'int32') new_shape = [input_shape[0], input_shape[1], y2 - y1, x2 - x1] x_crop = x[:, :, y1:y2, x1:x2] xm = K.reshape(x_crop, new_shape) pooled_val = K.max(xm, axis=(2, 3)) outputs.append(pooled_val) elif self.dim_ordering == 'tf': for pool_num, num_pool_regions in enumerate(self.pool_list): for ix in range(num_pool_regions): for jy in range(num_pool_regions): x1 = ix * col_length[pool_num] x2 = ix * col_length[pool_num] + col_length[pool_num] y1 = jy * row_length[pool_num] y2 = jy * row_length[pool_num] + row_length[pool_num] x1 = K.cast(K.round(x1), 'int32') x2 = K.cast(K.round(x2), 'int32') y1 = K.cast(K.round(y1), 'int32') y2 = K.cast(K.round(y2), 'int32') new_shape = [input_shape[0], y2 - y1, x2 - x1, input_shape[3]] x_crop = x[:, y1:y2, x1:x2, :] xm = K.reshape(x_crop, new_shape) pooled_val = K.max(xm, axis=(1, 2)) outputs.append(pooled_val) outputs = K.concatenate(outputs) return outputs
def test_merge(): from keras.layers import Input, merge from keras.models import Model # test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'. input_shapes = [(3, 2), (3, 2)] inputs = [np.random.random(shape) for shape in input_shapes] # test functional API for mode in ['sum', 'mul', 'concat', 'ave']: print(mode) input_a = Input(shape=input_shapes[0][1:]) input_b = Input(shape=input_shapes[1][1:]) merged = merge([input_a, input_b], mode=mode) model = Model([input_a, input_b], merged) model.compile('rmsprop', 'mse') expected_output_shape = model.get_output_shape_for(input_shapes) actual_output_shape = model.predict(inputs).shape assert expected_output_shape == actual_output_shape config = model.get_config() model = Model.from_config(config) model.compile('rmsprop', 'mse') # test lambda with output_shape lambda input_a = Input(shape=input_shapes[0][1:]) input_b = Input(shape=input_shapes[1][1:]) merged = merge([input_a, input_b], mode=lambda tup: K.concatenate([tup[0], tup[1]]), output_shape=lambda tup: (tup[0][:-1],) + (tup[0][-1] + tup[1][-1],)) expected_output_shape = model.get_output_shape_for(input_shapes) actual_output_shape = model.predict(inputs).shape assert expected_output_shape == actual_output_shape config = model.get_config() model = Model.from_config(config) model.compile('rmsprop', 'mse') # test function with output_shape function def fn_mode(tup): x, y = tup return K.concatenate([x, y]) def fn_output_shape(tup): s1, s2 = tup return (s1[:-1],) + (s1[-1] + s2[-1],) input_a = Input(shape=input_shapes[0][1:]) input_b = Input(shape=input_shapes[1][1:]) merged = merge([input_a, input_b], mode=fn_mode, output_shape=fn_output_shape) expected_output_shape = model.get_output_shape_for(input_shapes) actual_output_shape = model.predict(inputs).shape assert expected_output_shape == actual_output_shape config = model.get_config() model = Model.from_config(config) model.compile('rmsprop', 'mse')
def set_batch_function(self, model, input_shape, batch_size, nb_actions, gamma): input_dim = np.prod(input_shape) samples = K.placeholder(shape=(batch_size, input_dim * 2 + 3)) S = samples[:, 0 : input_dim] a = samples[:, input_dim] a = K.cast(a, '') r = samples[:, input_dim + 1] S_prime = samples[:, input_dim + 2 : 2 * input_dim + 2] game_over = samples[:, 2 * input_dim + 2 : 2 * input_dim + 3] r = K.reshape(r, (batch_size, 1)) r = K.repeat(r, nb_actions) r = K.reshape(r, (batch_size, nb_actions)) game_over = K.repeat(game_over, nb_actions) game_over = K.reshape(game_over, (batch_size, nb_actions)) S = K.reshape(S, (batch_size, ) + input_shape) S_prime = K.reshape(S_prime, (batch_size, ) + input_shape) X = K.concatenate([S, S_prime], axis=0) Y = model(X) Qsa = K.max(Y[batch_size:], axis=1) Qsa = K.reshape(Qsa, (batch_size, 1)) Qsa = K.repeat(Qsa, nb_actions) Qsa = K.reshape(Qsa, (batch_size, nb_actions)) delta = K.reshape(self.one_hot(a, nb_actions), (batch_size, nb_actions)) targets = (1 - delta) * Y[:batch_size] + delta * (r + gamma * (1 - game_over) * Qsa) self.batch_function = K.function(inputs=[samples], outputs=[S, targets])
def call(self, x): assert isinstance(x, list) inp_a, inp_b = x outp_a = K.l2_normalize(inp_a, -1) outp_b = K.l2_normalize(inp_b, -1) alpha = K.batch_dot(outp_b, outp_a, axes=[2, 2]) alpha = K.l2_normalize(alpha, 1) alpha = K.one_hot(K.argmax(alpha, 1), K.int_shape(inp_a)[1]) hmax = K.batch_dot(alpha, outp_b, axes=[1, 1]) kcon = K.eye(K.int_shape(inp_a)[1], dtype='float32') m = [] for i in range(self.output_dim): outp_a = inp_a * self.W[i] outp_hmax = hmax * self.W[i] outp_a = K.l2_normalize(outp_a, -1) outp_hmax = K.l2_normalize(outp_hmax, -1) outp = K.batch_dot(outp_hmax, outp_a, axes=[2, 2]) outp = K.sum(outp * kcon, -1, keepdims=True) m.append(outp) if self.output_dim > 1: persp = K.concatenate(m, 2) else: persp = m return [persp, persp]
def get_output(self, train=False): X = self.get_input(train) # 0,0,0,1,2,3,4 mask = self.get_input_mask(train) # 0,0,0,1,1,1,1 def reverse(x): rev = K.permute_dimensions(x, (1, 0, 2))[::-1] return K.permute_dimensions(rev, (1, 0, 2)) X_rev = reverse(X) # 4,3,2,1,0,0,0 Y = self.forward(X, mask) # 0,0,0,1,3,6,10 mask_rev = reverse(mask) if mask else None # 1,1,1,1,0,0,0 Y_rev = self.reverse(X_rev, mask_rev) # 4,7,9,10,10,10,10 #Fix allignment if self.return_sequences: Y_rev = reverse(Y_rev) # 10,10,10,10,9,7,4 if self.merge_mode == 'concat': return K.concatenate([Y, Y_rev]) elif self.merge_mode == 'sum': return Y + Y_rev elif self.merge_mode == 'ave': return (Y + Y_rev) / 2 elif self.merge_mode == 'mul': return Y * Y_rev
def recurrence(y_i, h): h_permute = K.permute_dimensions(h, [0, 2, 1]) # (batch_size, encoding_dim, input_length) e = K.l2_normalize( K.batch_dot(h_permute, s, axes=1), # (batch_size, input_length) axis=1) # (batch_size, input_length) # eqn 6 alpha = K.softmax(e) # (batch_size, input_length) # eqn 5 c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim) recurrence_result = K.expand_dims( K.concatenate([c, y_i], axis=1), dim=1) # (batch_size, 1, 2 * encoding_dim) expanded_h = Input(shape=(1, 2 * encoding_dim), name='expanded_h') gru = Sequential([ GRU(output_dim, return_sequences=False, input_shape=(1, 2 * encoding_dim)) ]) model = Model(input=[expanded_h], output=[gru(expanded_h)]) # (batch_size, 1, output_dim) return model(recurrence_result)
def test_lambda(): layer_test(layers.Lambda, kwargs={'function': lambda x: x + 1}, input_shape=(3, 2)) layer_test(layers.Lambda, kwargs={'function': lambda x, a, b: x * a + b, 'arguments': {'a': 0.6, 'b': 0.4}}, input_shape=(3, 2)) # test serialization with function def f(x): return x + 1 ld = layers.Lambda(f) config = ld.get_config() ld = deserialize_layer({'class_name': 'Lambda', 'config': config}) # test with lambda ld = layers.Lambda( lambda x: K.concatenate([K.square(x), x]), output_shape=lambda s: tuple(list(s)[:-1] + [2 * s[-1]])) config = ld.get_config() ld = layers.Lambda.from_config(config) # test serialization with output_shape function def f(x): return K.concatenate([K.square(x), x]) def f_shape(s): return tuple(list(s)[:-1] + [2 * s[-1]]) ld = layers.Lambda(f, output_shape=f_shape) config = ld.get_config() ld = deserialize_layer({'class_name': 'Lambda', 'config': config})
def main(weights_path, base_path, base_file, style_path, style_file, combo_path, img_width, img_height, iterations): result_prefix = base_file[:-4] + '_' + style_file[:-4] base_img_path = base_path + base_file style_img_path = style_path + style_file # get tensor representations of images base_img = K.variable(preprocess_image(base_img_path, img_width, img_height)) style_img = K.variable(preprocess_image(style_img_path, img_width, img_height)) combo_img = K.placeholder((1, 3, img_width, img_height)) # combine the 3 images into a single Keras tensor input_tensor = K.concatenate([base_img, style_img, combo_img], axis=0) print('Creating painting of {} in the style of {}'.format(base_file[:-4], style_file[:-4])) print('Loading model with VGG16 network weights...') nn = model(weights_path, input_tensor, img_width, img_height) loss, grads = calc_loss_grad(nn, combo_img, img_width, img_height) evaluate = Evaluator(loss, grads, combo_img, img_width, img_height) return optimizer(evaluate, img_width, img_height, combo_path, result_prefix, iterations=iterations)
def compute_mask(self, inputs, mask=None): if mask is None or not any([m is not None for m in mask]): return None assert hasattr(mask, '__len__') and len(mask) == len(inputs) if self.mode in ['sum', 'mul', 'ave']: bool_type = 'bool' if K._BACKEND == 'tensorflow' else 'int32' masks = [K.cast(m, bool_type) for m in mask if m is not None] mask = masks[0] for m in masks[1:]: mask = mask & m return mask elif self.mode in ['concat']: masks = [K.ones_like(inputs[i][:-1]) if m is None else m for i, m in zip(inputs, mask)] expanded_dims = [K.expand_dims(m) for m in masks] concatenated = K.concatenate(expanded_dims, axis=self.concat_axis) return K.all(concatenated, axis=-1, keepdims=False) elif self.mode in ['cos', 'dot']: return None elif hasattr(self.mode, '__call__'): if hasattr(self._output_mask, '__call__'): return self._output_mask(mask) else: return self._output_mask else: # this should have been caught earlier raise Exception('Invalid merge mode: {}'.format(self.mode))
def yolo_head(feats, anchors, num_classes, input_shape): """Convert final layer features to bounding box parameters.""" num_anchors = len(anchors) # Reshape to batch, height, width, num_anchors, box_params. anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], 1, 1]) grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, 1, 1]) grid = K.concatenate([grid_x, grid_y]) grid = K.cast(grid, K.dtype(feats)) feats = K.reshape( feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5]) box_xy = K.sigmoid(feats[..., :2]) box_wh = K.exp(feats[..., 2:4]) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:]) # Adjust preditions to each spatial grid point and anchor size. box_xy = (box_xy + grid) / K.cast(grid_shape[::-1], K.dtype(feats)) box_wh = box_wh * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats)) return box_xy, box_wh, box_confidence, box_class_probs
def simple_context(X, mask, n=activation_rnn_size): """Reduce the input just to its headline part (second half). For each word in this part it concatenate the output of the previous layer (RNN) with a weighted average of the outputs of the description part. In this only the last `rnn_size - activation_rnn_size` are used from each output. The first `activation_rnn_size` output is used to computer the weights for the averaging. """ desc, head = X[:, :maxlend, :], X[:, maxlend:, :] head_activations, head_words = head[:, :, :n], head[:, :, n:] desc_activations, desc_words = desc[:, :, :n], desc[:, :, n:] # RTFM http://deeplearning.net/software/theano/library/tensor/basic.html#theano.tensor.batched_tensordot # activation for every head word and every desc word activation_energies = K.batch_dot(head_activations, desc_activations, axes=(2, 2)) # make sure we dont use description words that are masked out activation_energies = activation_energies + -1e20 * K.expand_dims( 1. - K.cast(mask[:, :maxlend], 'float32'), 1) # for every head word compute weights for every desc word activation_energies = K.reshape(activation_energies, (-1, maxlend)) activation_weights = K.softmax(activation_energies) activation_weights = K.reshape(activation_weights, (-1, maxlenh, maxlend)) # for every head word compute weighted average of desc words desc_avg_word = K.batch_dot(activation_weights, desc_words, axes=(2, 1)) return K.concatenate((desc_avg_word, head_words))
def residual_drop(x, input_shape, output_shape, strides=(1, 1)): global add_tables nb_filter = output_shape[0] conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x) conv = BatchNormalization(axis=1)(conv) conv = Activation("relu")(conv) conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv) conv = BatchNormalization(axis=1)(conv) if strides[0] >= 2: x = AveragePooling2D(strides)(x) if (output_shape[0] - input_shape[0]) > 0: pad_shape = (1, output_shape[0] - input_shape[0], output_shape[1], output_shape[2]) padding = K.ones(pad_shape) padding = K.repeat_elements(padding, K.shape(x)[0], axis=0) x = Lambda(lambda y: K.concatenate([y, padding], axis=1), output_shape=output_shape)(x) _death_rate = K.variable(death_rate) scale = K.ones_like(conv) - _death_rate conv = Lambda(lambda c: K.in_test_phase(scale * c, c), output_shape=output_shape)(conv) out = merge([conv, x], mode="sum") out = Activation("relu")(out) gate = K.variable(1, dtype="uint8") add_tables += [{"death_rate": _death_rate, "gate": gate}] return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]), output_shape=output_shape)([out, x])
def get_output(self, train=False): print "Input Shape", self.input_shape print "ConvolutionDNASequenceBinding", self.output_shape X = self.get_input(train) if self.use_three_base_encoding: X_fwd = X[:,1:,:,:] X_rc = X[:,:3,:,:] else: X_fwd = X X_rc = X print self.W print self.b if self.W[1] is not None: W = self.W[0][self.W[1],:,:,:] else: W = self.W[0] if self.b[1] is not None: b = self.b[0][self.b[1]] else: b = self.b[0] fwd_rv = K.conv2d(X_fwd, W, border_mode='valid') \ + K.reshape(b, (1, self.nb_motifs, 1, 1)) rc_rv = K.conv2d(X_rc, W[:,::-1,:,::-1], border_mode='valid') \ + K.reshape(b, (1, self.nb_motifs, 1, 1)) rv = K.concatenate((fwd_rv, rc_rv), axis=2) #return rv.dimshuffle((0,3,2,1)) return rv # K.permute_dimensions(rv, (0,3,2,1))
def get_output(self, train): x = self.get_input(train) x -= K.mean(x, axis=1, keepdims=True) x = K.l2_normalize(x, axis=1) pos = K.relu(x) neg = K.relu(-x) return K.concatenate([pos, neg], axis=1)
return x # endregion # region load the pretrained VGG19 print('loading model...') target_image = K.constant(preprocess_image(target_image_path)) style_reference_image = K.constant( preprocess_image(style_reference_image_path)) # This placeholder will contain our generated image combination_image = K.placeholder((1, img_height, img_width, 3)) # We combine the 3 images into a single batch input_tensor = K.concatenate( [target_image, style_reference_image, combination_image], axis=0) # We build the VGG19 network with our batch of 3 images as input. # The model will be loaded with pre-trained ImageNet weights. model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) print('Model loaded.') # endregion # region content loss def content_loss(base, combination): return K.sum(K.square((combination - base)))
yield [x, y], z x, y = [], [] _ = 0 #CBOW输入 input_words = Input(shape=(window * 2, ), dtype='int32') input_vecs = Embedding(nb_word, word_size, name='word2vec')(input_words) input_vecs_sum = Lambda(lambda x: K.sum(x, axis=1))( input_vecs) #CBOW模型,直接将上下文词向量求和 #构造随机负样本,与目标组成抽样 target_word = Input(shape=(1, ), dtype='int32') negatives = Lambda(lambda x: K.random_uniform( (K.shape(x)[0], nb_negative), 0, nb_word, 'int32'))(target_word) samples = Lambda(lambda x: K.concatenate(x))( [target_word, negatives]) #构造抽样,负样本随机抽。负样本也可能抽到正样本,但概率小。 #只在抽样内做Dense和softmax softmax_weights = Embedding(nb_word, word_size, name='W')(samples) softmax_biases = Embedding(nb_word, 1, name='b')(samples) softmax = Lambda(lambda x: K.softmax( (K.batch_dot(x[0], K.expand_dims(x[1], 2)) + x[2])[:, :, 0]))( [softmax_weights, input_vecs_sum, softmax_biases]) #用Embedding层存参数,用K后端实现矩阵乘法,以此复现Dense层的功能 #留意到,我们构造抽样时,把目标放在了第一位,也就是说,softmax的目标id总是0,这可以从data_generator中的z变量的写法可以看出 model = Model(inputs=[input_words, target_word], outputs=softmax) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam',
def lnln_model_flip(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=True): reg_val = 1 v_leak = 0 v_exc = 60 v_inh = -30 g_leak = 1 filter_shape[1] = 1 assert (np.mod(num_filter, 2) == 0) num_filter = int(num_filter / 2) pad_x = int((filter_shape[1] - 1)) + 2 pad_t = int((filter_shape[0] - 1)) # Define the input as a tensor with shape input_shape image_in = Input(input_shape) s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in) s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in) s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in) g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt) g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt) g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt) g2_both = g2(s2) g1_1 = g1(s1) g1_2 = g1(s3) g3_1 = g3(s3) g3_2 = g3(s1) expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1)) squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1)) g2_both = expand_last(g2_both) g1_1 = expand_last(g1_1) g1_2 = expand_last(g1_2) g3_1 = expand_last(g3_1) g3_2 = expand_last(g3_2) numerator_in_1 = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1_1, g2_both, g3_1]) numerator_in_2 = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1_2, g2_both, g3_2]) numerator_comb = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False) vm_1 = numerator_comb(numerator_in_1) vm_2 = numerator_comb(numerator_in_2) vm_1 = squeeze_last(vm_1) vm_2 = squeeze_last(vm_2) bias_layer = BiasLayer() vm_1_bias = bias_layer(vm_1) vm_2_bias = bias_layer(vm_2) vm_1_rect = Lambda(lambda lam: K.relu(lam))(vm_1_bias) vm_2_rect = Lambda(lambda lam: K.relu(lam))(vm_2_bias) vm = subtract([vm_1_rect, vm_2_rect]) if sum_over_space: conv_x_size = vm.get_shape().as_list()[2] else: conv_x_size = 1 combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False)(vm) # Create model model = Model(inputs=image_in, outputs=combine_filters, name='lnln_model_flip') return model, pad_x, pad_t
def zeropad(x): y = K.zeros_like(x) return K.concatenate([x, y], axis=2)
def custom_loss(y_true, y_pred): return -K.mean( self.critic( K.concatenate( [input_tensor, self.actor(input_tensor)], axis=1)))
def step(self, x, states, training=None): h_tm1 = states[0] c_tm1 = states[1] x_seq = states[2] # repeat the hidden state to the length of the sequence _htm = K.repeat(h_tm1, self.time_step_e) #(batch,time_step,units) # concatenate a(previus output lstm) + hidden state concatenate = K.concatenate( [_htm, x_seq], axis=-1) #(batch,time_step,h_units+x_seq_units) # now multiplty the weight matrix with the repeated hidden state # apply the a dense layer over the time dimension of the sequence # do it here because it doesn't depend on any previous steps # thefore we can save computation time: dot_dense = time_distributed_dense( concatenate, self.W_cx, b=self.b_cx, input_dim=self.input_dim_e + self.units, timesteps=self.time_step_e, output_dim=self.atten_units) #(samples,timestep,atten_units) # we need to supply the full sequence of inputs to step (as the attention_vector) # calculate the attention probabilities # this relates how much other timesteps contributed to this one. et = K.dot( K.relu(dot_dense), #(batch,time_step,atten_units) K.expand_dims(self.C_cx)) at = K.exp(et) #(batch,time_step,1) at_sum = K.cast(K.sum(at, axis=1) + K.epsilon(), K.floatx()) #(batch,1) at_sum_repeated = K.repeat(at_sum, self.time_step_e) #(batch,time_step,1) at /= at_sum_repeated # vector of size (batchsize, time_steps, 1) # calculate the context vector context = K.squeeze(K.batch_dot(at, x_seq, axes=1), axis=1) #(batchsize,input_dim) if 0 < self.dropout < 1 and self._dropout_mask is None: self._dropout_mask = _generate_dropout_mask(K.ones_like(context), self.dropout, training=training, count=4) if (0 < self.recurrent_dropout < 1 and self._recurrent_dropout_mask is None): self._recurrent_dropout_mask = _generate_dropout_mask( K.ones_like(states[0]), self.recurrent_dropout, training=training, count=4) # dropout matrices for input units B_W = self._dropout_mask # dropout matrices for recurrent units B_U = self._recurrent_dropout_mask # ~~~> calculate new hidden state yhat_i = K.dot(x, self.V_i) #(batchsize,units) yhat_f = K.dot(x, self.V_f) #(batchsize,units) yhat_c = K.dot(x, self.V_c) #(batchsize,units) yhat_o = K.dot(x, self.V_o) #(batchsize,units) if 0 < self.dropout < 1.: x_i = K.dot(context * B_W[0], self.W_i) + self.b_i #(batchsize,units) x_f = K.dot(context * B_W[1], self.W_f) + self.b_f #(batchsize,units) x_c = K.dot(context * B_W[2], self.W_c) + self.b_c #(batchsize,units) x_o = K.dot(context * B_W[3], self.W_o) + self.b_o #(batchsize,units) else: x_i = K.dot(context, self.W_i) + self.b_i #(batchsize,units) x_f = K.dot(context, self.W_f) + self.b_f #(batchsize,units) x_c = K.dot(context, self.W_c) + self.b_c #(batchsize,units) x_o = K.dot(context, self.W_o) + self.b_o #(batchsize,units) if 0 < self.recurrent_dropout < 1.: h_tm1_i = K.dot(h_tm1 * B_U[0], self.U_i) #(batchsize,units) h_tm1_f = K.dot(h_tm1 * B_U[1], self.U_f) #(batchsize,units) h_tm1_c = K.dot(h_tm1 * B_U[2], self.U_c) #(batchsize,units) h_tm1_o = K.dot(h_tm1 * B_U[3], self.U_o) #(batchsize,units) else: h_tm1_i = K.dot(h_tm1, self.U_i) #(batchsize,units) h_tm1_f = K.dot(h_tm1, self.U_f) #(batchsize,units) h_tm1_c = K.dot(h_tm1, self.U_c) #(batchsize,units) h_tm1_o = K.dot(h_tm1, self.U_o) #(batchsize,units) i = self.recurrent_activation(x_i + h_tm1_i + yhat_i) #(batchsize,units) f = self.recurrent_activation(x_f + h_tm1_f + yhat_f) #(batchsize,units) o = self.recurrent_activation(x_o + h_tm1_o + yhat_o) #(batchsize,units) c_ = self.activation(x_c + h_tm1_c + yhat_c) #(batchsize,units) c = f * c_tm1 + i * c_ h = o * self.activation(c) #(batchsize,units) if 0 < self.dropout + self.recurrent_dropout: if training is None: h._uses_learning_phase = True #apply maxout layer with dropout maxout = self.max_out(inputs=K.dot(h, self.U_p), num_units=self.gmax) drop = Dropout(0.3)(maxout) #apply softmax _y_hat = activations.softmax(K.dot(drop, self.M_p) + self.b_p) if self.return_probabilities: return at, [h, c] else: return _y_hat, [h, c]
def call(self, x, mask=None): input_shape = K.shape(x) if self.dim_ordering == 'th': num_rows = input_shape[2] num_cols = input_shape[3] elif self.dim_ordering == 'tf': num_rows = input_shape[1] num_cols = input_shape[2] row_length = [K.cast(num_rows, 'float32') / i for i in self.pool_list] col_length = [K.cast(num_cols, 'float32') / i for i in self.pool_list] outputs = [] if self.dim_ordering == 'th': for pool_num, num_pool_regions in enumerate(self.pool_list): for jy in range(num_pool_regions): for ix in range(num_pool_regions): x1 = ix * col_length[pool_num] x2 = ix * col_length[pool_num] + col_length[pool_num] y1 = jy * row_length[pool_num] y2 = jy * row_length[pool_num] + row_length[pool_num] x1 = K.cast(K.round(x1), 'int32') x2 = K.cast(K.round(x2), 'int32') y1 = K.cast(K.round(y1), 'int32') y2 = K.cast(K.round(y2), 'int32') new_shape = [input_shape[0], input_shape[1], y2 - y1, x2 - x1] x_crop = x[:, :, y1:y2, x1:x2] xm = K.reshape(x_crop, new_shape) pooled_val = K.max(xm, axis=(2, 3)) outputs.append(pooled_val) elif self.dim_ordering == 'tf': for pool_num, num_pool_regions in enumerate(self.pool_list): for jy in range(num_pool_regions): for ix in range(num_pool_regions): x1 = ix * col_length[pool_num] x2 = ix * col_length[pool_num] + col_length[pool_num] y1 = jy * row_length[pool_num] y2 = jy * row_length[pool_num] + row_length[pool_num] x1 = K.cast(K.round(x1), 'int32') x2 = K.cast(K.round(x2), 'int32') y1 = K.cast(K.round(y1), 'int32') y2 = K.cast(K.round(y2), 'int32') new_shape = [input_shape[0], y2 - y1, x2 - x1, input_shape[3]] x_crop = x[:, y1:y2, x1:x2, :] xm = K.reshape(x_crop, new_shape) pooled_val = K.max(xm, axis=(1, 2)) outputs.append(pooled_val) if self.dim_ordering == 'th': outputs = K.concatenate(outputs) elif self.dim_ordering == 'tf': #outputs = K.concatenate(outputs,axis = 1) outputs = K.concatenate(outputs) #outputs = K.reshape(outputs,(len(self.pool_list),self.num_outputs_per_channel,input_shape[0],input_shape[1])) #outputs = K.permute_dimensions(outputs,(3,1,0,2)) #outputs = K.reshape(outputs,(input_shape[0], self.num_outputs_per_channel * self.nb_channels)) return outputs
# this will contain our generated image if K.image_dim_ordering() == 'th': combination_image = K.placeholder((1, 3, img_width, img_height)) else: combination_image = K.placeholder((1, img_width, img_height, 3)) image_tensors = [base_image] for style_image_tensor in style_reference_images: image_tensors.append(style_image_tensor) image_tensors.append(combination_image) nb_tensors = len(image_tensors) nb_style_images = nb_tensors - 2 # Content and Output image not considered # combine the various images into a single Keras tensor input_tensor = K.concatenate(image_tensors, axis=0) if K.image_dim_ordering() == "th": shape = (nb_tensors, 3, img_width, img_height) else: shape = (nb_tensors, img_width, img_height, 3) ip = Input(tensor=input_tensor, batch_shape=shape) # build the VGG16 network with our 3 images as input x = Convolution2D(64, (3, 3), activation='relu', name='conv1_1', padding='same')(ip) x = Convolution2D(64, (3, 3), activation='relu',
def concat_tensors(tensors, axis=-1): return K.concatenate([K.expand_dims(t, axis=axis) for t in tensors])
def call(self, inputs): if self.r_num == 1: # if there is no routing (and this is so when r_num is 1 and all c are equal) # then this is a common convolution outputs = K.conv2d( K.reshape(inputs, (-1, self.h_i, self.w_i, self.ch_i * self.n_i)), K.reshape( self.w, self.kernel_size + (self.ch_i * self.n_i, self.ch_j * self.n_j)), data_format='channels_last', strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate) outputs = squeeze( K.reshape(outputs, ((-1, self.h_j, self.w_j, self.ch_j, self.n_j)))) else: bt = K.shape(inputs)[0] ksz = self.kernel_size[0] * self.kernel_size[1] xr = K.reshape(inputs, (-1, self.h_i, self.w_i, self.ch_i * self.n_i)) pt = tf.extract_image_patches(xr, (1, ) + self.kernel_size + (1, ), (1, ) + self.strides + (1, ), ( 1, 1, 1, 1, ), 'VALID') pt = K.reshape(pt, (-1, ksz * self.ch_i, self.n_i)) wr = K.reshape(self.w, (ksz * self.ch_i, self.n_i, self.ch_j * self.n_j)) global useGPU # it sometimes works faster on GPU when batch is devided into two parts # bp = K.expand_dims(bt // 2, axis=0) if useGPU else K.constant([2], dtype=tf.int32) bp = K.expand_dims(bt // 1, axis=0) if useGPU else K.constant( [2], dtype=tf.int32) if self.strides != (1, 1): zr_shape = K.concatenate([ bp, K.constant([self.w_j, ksz * self.ch_i * self.ch_j], dtype=tf.int32) ]) zr = tf.zeros(shape=zr_shape) zc_shape = K.concatenate([ bp, K.constant([self.ah_j, ksz * self.ch_i * self.ch_j], dtype=tf.int32) ]) zc = tf.zeros(shape=zc_shape) def rt(ptb): ptb = K.reshape(ptb, (-1, ksz * self.ch_i, self.n_i)) if useGPU: ub = tf.einsum('bin,inj->bij', ptb, wr) else: ul = [] for i in range(ksz * self.ch_i): ul.append(K.dot(ptb[:, i], wr[i])) ub = K.stack(ul, axis=1) #b = tf.constant_initializer(0.)((bp, self.h_i*self.w_i*self.ch_i, # ksz * self.ch_j)) b = 0.0 j_all = self.h_j * self.w_j * self.ch_j j_add = j_all - ksz * self.ch_j for r in range(self.r_num): ex = K.exp(b * self.b_alphas[r]) if r > 0: c = ex / ( (K.sum(ex, axis=-1, keepdims=True) + K.epsilon()) + j_add) * j_all c = K.reshape(c, (-1, self.h_i, self.w_i, self.ch_i * ksz * self.ch_j)) c = K.stop_gradient(c) pc = tf.extract_image_patches( c, (1, ) + self.kernel_size + (1, ), (1, ) + self.strides + (1, ), ( 1, 1, 1, 1, ), 'VALID') pc = K.reshape(pc, (-1, self.h_j, self.w_j, ksz, self.ch_i, self.kernel_size[0] * self.kernel_size[1], self.ch_j)) pcl = [] for n in range(ksz): pcl.append( pc[:, :, :, n, :, self.kernel_size[0] * self.kernel_size[1] - 1 - n]) pcc = K.stack(pcl, axis=3) if useGPU: pcc = K.reshape(pcc, (-1, self.h_j * self.w_j * ksz * self.ch_i * self.ch_j, 1)) ub = K.reshape(ub, (-1, self.h_j * self.w_j * ksz * self.ch_i * self.ch_j, self.n_j)) cu = pcc * ub else: pcc = K.reshape(pcc, (-1, 1)) ub = K.reshape(ub, (-1, self.n_j, 1)) cul = [] for n in range(self.n_j): cul.append(ub[:, n] * pcc) cu = K.stack(cul, axis=-2) else: cu = ub cu = K.reshape(cu, (-1, self.h_j * self.w_j, ksz * self.ch_i, self.ch_j, self.n_j)) s = K.sum(cu, axis=-3) v = squeeze(s) if r == self.r_num - 1: break v = K.stop_gradient(v) ubr = K.reshape(K.stop_gradient(ub), (-1, self.h_j * self.w_j, ksz * self.ch_i, self.ch_j, self.n_j)) if True: #if useGPU: a = tf.einsum('bjck,bjick->bjic', v, ubr) else: al = [] for i in range(ksz * self.ch_i): al.append( K.batch_dot( K.reshape(ubr[:, :, i], (-1, self.h_j * self.w_j * self.ch_j, 1, self.n_j)), K.reshape(v, (-1, self.h_j * self.w_j * self.ch_j, self.n_j, 1)))) a = K.stack(al, axis=1) a = K.reshape(a, (-1, ksz * self.ch_i, self.h_j * self.w_j, self.ch_j)) a = K.permute_dimensions(a, [0, 2, 1, 3]) ph, pw = 2, 2 a = K.reshape( a, (-1, self.h_j, self.w_j, ksz * self.ch_i * self.ch_j)) if self.strides == (1, 1): aa = a else: rl = [] for r in range(self.ah_j): rl.append(zr if r % ph else a[:, r // ph]) rs = K.stack(rl, axis=1) cl = [] for c in range(self.aw_j): cl.append(zc if c % pw else rs[:, :, c // pw]) aa = K.stack(cl, axis=-2) aa = K.spatial_2d_padding(aa, ((ph, ph), (pw, pw)), data_format='channels_last') pa = tf.extract_image_patches( aa, (1, ) + self.kernel_size + (1, ), ( 1, 1, 1, 1, ), #(1,)+strides+(1,), ( 1, 1, 1, 1, ), 'VALID') pa = K.reshape(pa, (-1, self.h_i * self.w_i, ksz, ksz, self.ch_i, self.ch_j)) pal = [] for n in range(ksz): pal.append(pa[:, :, n, ksz - 1 - n]) paa = K.stack(pal, axis=3) paa = K.reshape( paa, (-1, self.h_i * self.w_i * self.ch_i, ksz * self.ch_j)) b = b + paa return v v = tf.map_fn(rt, K.reshape(pt, (-1, bp[0], self.h_j, self.w_j, ksz * self.ch_i, self.n_i)), parallel_iterations=100, back_prop=True, infer_shape=False) outputs = v outputs = K.reshape(outputs, (-1, self.h_j, self.w_j, self.ch_j, self.n_j)) return outputs
# Create tensor variables for images if K.image_dim_ordering() == 'th': shape = (1, nb_colors, img_nrows, img_ncols) else: shape = (1, img_nrows, img_ncols, nb_colors) style_image = K.variable(preprocess_image(style_img_path)) target_image = K.placeholder(shape=shape) if use_content_img: content_image = K.variable(preprocess_image(content_img_path)) else: content_image = K.zeros(shape=shape) images = K.concatenate([style_image, target_image, content_image], axis=0) # Create tensor variables for masks raw_style_mask, raw_target_mask = load_mask_labels() style_mask = K.variable(raw_style_mask.astype("float32")) target_mask = K.variable(raw_target_mask.astype("float32")) masks = K.concatenate([style_mask, target_mask], axis=0) # index constants for images and tasks variables STYLE, TARGET, CONTENT = 0, 1, 2 # Build image model, mask model and use layer outputs as features # image model as VGG19 image_model = vgg19.VGG19(include_top=False, input_tensor=images) # mask model as a series of pooling
#for target user h_i = decoder_pred h_all = h_i if cfg.use_embedding_AME: W_i = W_i_pool[-1] u_i_list.append(W_i(h_all)) V_i = V_i_pool[-1] u_si_list.append(V_i(h_i)) else: u_i_list.append(h_all) u_si_list.append(h_i) assert len(u_i_list)==num_user assert len(u_si_list)==num_user u_i_list = Lambda(lambda x: K.concatenate(x, axis=1))(u_i_list)#(batch,34,256) u_si_list = Lambda(lambda x: K.concatenate(x, axis=1))(u_si_list)#(batch,34,256) alpha_list = ui_usi_similarity(u_i_list,u_si_list) outputs = merge_experts_contribution(alpha_list,concat_outputs) all_outputs.append(outputs) inputs = outputs ## Concatenate all predictions decoder_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs) model = Model([encoder_inputs, others_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer='Adam', loss='mean_squared_error',metrics=['accuracy'])
def call(self, inputs, mask=None): ''' :param inputs: a list of tensor of length not larger than 2, or a memory tensor of size BxTXD1. If a list, the first entry is memory, and the second one is query tensor of size BxD2 if any :param mask: the masking entry will be directly discarded :return: a tensor of size BxD1, weighted summing along the sequence dimension ''' query = None if isinstance(inputs, list): memory = inputs[0] if len(inputs) > 1: query = inputs[1] elif len(inputs) > 2: raise ValueError('inputs length should not be larger than 2') if isinstance(mask, list): mask = mask[0] else: memory = inputs input_shape = K.int_shape(memory) if len(input_shape) > 3: input_length = input_shape[1] memory = K.reshape(memory, (-1, ) + input_shape[2:]) if mask is not None: mask = K.reshape(mask, (-1, ) + input_shape[2:-1]) if query is not None: raise ValueError('query can be not supported') last = memory[:, -1, :] memory = memory[:, :-1, :] if query is None: query = last else: query = K.concatenate([query, last], axis=-1) if self.method is None: if len(input_shape) > 3: output_shape = K.int_shape(last) return K.reshape(last, (-1, input_shape[1], output_shape[-1])) else: return last elif self.method == 'cba': hidden = K.dot(memory, self.Wh) + K.expand_dims( K.dot(query, self.Wq), 1) hidden = K.tanh(hidden) s = K.squeeze(K.dot(hidden, self.v), -1) elif self.method == 'ga': s = K.sum(K.expand_dims(K.dot(query, self.Wq), 1) * memory, axis=-1) else: s = K.squeeze(K.dot(memory, self.v), -1) s = K.softmax(s) if mask is not None: mask = mask[:, :-1] s *= K.cast(mask, dtype='float32') sum_by_time = K.sum(s, axis=-1, keepdims=True) s = s / (sum_by_time + K.epsilon()) #return [K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1), s] result = K.concatenate( [K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1) if len(input_shape) > 3: output_shape = K.int_shape(result) return K.reshape(result, (-1, input_shape[1], output_shape[-1])) else: return result
content_array[:, :, :, 0] -= 103.939 content_array[:, :, :, 1] -= 116.779 content_array[:, :, :, 2] -= 123.68 style_array[:, :, :, 0] -= 103.939 style_array[:, :, :, 1] -= 116.779 style_array[:, :, :, 2] -= 123.68 height = 512 width = 512 content_image = backend.variable(content_array) style_image = backend.variable(style_array) combination_image = backend.placeholder((1, height, width, 3)) input_tensor = backend.concatenate( [content_image, style_image, combination_image], axis=0) model = VGG16(input_tensor=input_tensor, weights='imagenet', include_top=False) model.summary() content_weight = backend.variable(0.05) style_weight = backend.variable(50.0) total_variation_weight = backend.variable(1.0) layers = dict([(layer.name, layer.output) for layer in model.layers]) print(layers) loss = backend.variable(0.) def content_loss(content, combination):
def build(self, input_shape): self.T = input_shape[1] if self.first: self.input_dim = input_shape[2] else: self.input_dim = input_shape[2] - 1 self.input_spec = [InputSpec(shape=input_shape)] self.states = [None, None, None] self.W_i = self.init((self.input_dim, self.output_dim), name='{}_W_i'.format(self.name)) self.U_i = self.inner_init((self.output_dim, self.output_dim), name='{}_U_i'.format(self.name)) self.b_i = K.zeros((self.output_dim, ), name='{}_b_i'.format(self.name)) self.W_f = self.init((self.input_dim, self.output_dim), name='{}_W_f'.format(self.name)) self.U_f = self.inner_init((self.output_dim, self.output_dim), name='{}_U_f'.format(self.name)) self.b_f = self.forget_bias_init((self.output_dim, ), name='{}_b_f'.format(self.name)) self.W_c = self.init((self.input_dim, self.output_dim), name='{}_W_c'.format(self.name)) self.U_c = self.inner_init((self.output_dim, self.output_dim), name='{}_U_c'.format(self.name)) self.b_c = K.zeros((self.output_dim, ), name='{}_b_c'.format(self.name)) self.W_o = self.init((self.input_dim, self.output_dim), name='{}_W_o'.format(self.name)) self.U_o = self.inner_init((self.output_dim, self.output_dim), name='{}_U_o'.format(self.name)) self.b_o = K.zeros((self.output_dim, ), name='{}_b_o'.format(self.name)) self.W_s = self.init((self.input_dim, 1), name='{}_W_s'.format(self.name)) self.U_s = self.inner_init((self.output_dim, 1), name='{}_U_s'.format(self.name)) self.b_s = K.zeros((1, ), name='{}_b_s'.format(self.name)) self.regularizers = [] if self.W_regularizer: self.W_regularizer.set_param( K.concatenate( [self.W_i, self.W_f, self.W_c, self.W_o, self.W_s])) self.regularizers.append(self.W_regularizer) if self.U_regularizer: self.U_regularizer.set_param( K.concatenate( [self.U_i, self.U_f, self.U_c, self.U_o, self.U_s])) self.regularizers.append(self.U_regularizer) if self.b_regularizer: self.b_regularizer.set_param( K.concatenate( [self.b_i, self.b_f, self.b_c, self.b_o, self.b_s])) self.regularizers.append(self.b_regularizer) self.trainable_weights = [ self.W_i, self.U_i, self.b_i, self.W_c, self.U_c, self.b_c, self.W_f, self.U_f, self.b_f, self.W_o, self.U_o, self.b_o, self.W_s, self.U_s, self.b_s ]
def yolo_loss(args, anchors, num_classes, rescore_confidence=False, print_loss=False): """YOLO localization loss function. Parameters ---------- yolo_output : tensor Final convolutional layer features. true_boxes : tensor Ground truth boxes tensor with shape [batch, num_true_boxes, 5] containing box x_center, y_center, width, height, and class. detectors_mask : array 0/1 mask for detector positions where there is a matching ground truth. matching_true_boxes : array Corresponding ground truth boxes for positive detector positions. Already adjusted for conv height and width. anchors : tensor Anchor boxes for model. num_classes : int Number of object classes. rescore_confidence : bool, default=False If true then set confidence target to IOU of best predicted box with the closest matching ground truth box. print_loss : bool, default=False If True then use a tf.Print() to print the loss components. Returns ------- mean_loss : float mean localization loss across minibatch """ (yolo_output, true_boxes, detectors_mask, matching_true_boxes) = args num_anchors = len(anchors) object_scale = 5 no_object_scale = 1 class_scale = 1 coordinates_scale = 1 pred_xy, pred_wh, pred_confidence, pred_class_prob = yolo_head( yolo_output, anchors, num_classes) # Unadjusted box predictions for loss. # TODO: Remove extra computation shared with yolo_head. yolo_output_shape = K.shape(yolo_output) feats = K.reshape(yolo_output, [ -1, yolo_output_shape[1], yolo_output_shape[2], num_anchors, num_classes + 5 ]) pred_boxes = K.concatenate( (K.sigmoid(feats[..., 0:2]), feats[..., 2:4]), axis=-1) # TODO: Adjust predictions by image width/height for non-square images? # IOUs may be off due to different aspect ratio. # Expand pred x,y,w,h to allow comparison with ground truth. # batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params pred_xy = K.expand_dims(pred_xy, 4) pred_wh = K.expand_dims(pred_wh, 4) pred_wh_half = pred_wh / 2. pred_mins = pred_xy - pred_wh_half pred_maxes = pred_xy + pred_wh_half true_boxes_shape = K.shape(true_boxes) # batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params true_boxes = K.reshape(true_boxes, [ true_boxes_shape[0], 1, 1, 1, true_boxes_shape[1], true_boxes_shape[2] ]) true_xy = true_boxes[..., 0:2] true_wh = true_boxes[..., 2:4] # Find IOU of each predicted box with each ground truth box. true_wh_half = true_wh / 2. true_mins = true_xy - true_wh_half true_maxes = true_xy + true_wh_half intersect_mins = K.maximum(pred_mins, true_mins) intersect_maxes = K.minimum(pred_maxes, true_maxes) intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.) intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] pred_areas = pred_wh[..., 0] * pred_wh[..., 1] true_areas = true_wh[..., 0] * true_wh[..., 1] union_areas = pred_areas + true_areas - intersect_areas iou_scores = intersect_areas / union_areas # Best IOUs for each location. best_ious = K.max(iou_scores, axis=4) # Best IOU scores. best_ious = K.expand_dims(best_ious) # A detector has found an object if IOU > thresh for some true box. object_detections = K.cast(best_ious > 0.6, K.dtype(best_ious)) # TODO: Darknet region training includes extra coordinate loss for early # training steps to encourage predictions to match anchor priors. # Determine confidence weights from object and no_object weights. # NOTE: YOLO does not use binary cross-entropy here. no_object_weights = (no_object_scale * (1 - object_detections) * (1 - detectors_mask)) no_objects_loss = no_object_weights * K.square(-pred_confidence) if rescore_confidence: objects_loss = (object_scale * detectors_mask * K.square(best_ious - pred_confidence)) else: objects_loss = (object_scale * detectors_mask * K.square(1 - pred_confidence)) confidence_loss = objects_loss + no_objects_loss # Classification loss for matching detections. # NOTE: YOLO does not use categorical cross-entropy loss here. matching_classes = K.cast(matching_true_boxes[..., 4], 'int32') matching_classes = K.one_hot(matching_classes, num_classes) classification_loss = (class_scale * detectors_mask * K.square(matching_classes - pred_class_prob)) # Coordinate loss for matching detection boxes. matching_boxes = matching_true_boxes[..., 0:4] coordinates_loss = (coordinates_scale * detectors_mask * K.square(matching_boxes - pred_boxes)) confidence_loss_sum = K.sum(confidence_loss) classification_loss_sum = K.sum(classification_loss) coordinates_loss_sum = K.sum(coordinates_loss) total_loss = 0.5 * ( confidence_loss_sum + classification_loss_sum + coordinates_loss_sum) if print_loss: total_loss = tf.Print( total_loss, [ total_loss, confidence_loss_sum, classification_loss_sum, coordinates_loss_sum ], message='yolo_loss, conf_loss, class_loss, box_coord_loss:') return total_loss
def attributes_update(self, attributes, depth, graph, original_graph, bonds): '''Given the current attributes, the current depth, and the graph that the attributes are based on, this function will update the 2D attributes tensor''' ############# GET NEW ATTRIBUTE MATRIX ######################### # New pre-activated attribute matrix v = M_i,j,: x ones((N_atom, 1)) -> (N_atom, N_features) # as long as dimensions are appropriately shuffled shuffled_graph = graph.copy().dimshuffle( (2, 0, 1)) # (N_feature x N_atom x N_atom) shuffled_graph.name = 'shuffled_graph' ones_vec = K.ones_like(attributes[:, 0]) # (N_atom x 1) ones_vec.name = 'ones_vec' (new_preactivated_attributes, updates) = theano.scan( lambda x: K.dot(x, ones_vec), sequences=shuffled_graph) # (N_features x N_atom) # Need to pass through an activation function still # Final attribute = bond flag = is not part of W_inner or b_inner (new_attributes, updates) = theano.scan(lambda x: self.activation_inner( K.dot(x, self.W_inner[depth, :, :]) + self.b_inner[depth, 0, :]), sequences=new_preactivated_attributes[:-1, :].T ) # (N_atom x N_features -1) # Append last feature (bond flag) after the loop new_attributes = K.concatenate((new_attributes, attributes[:, -1:]), axis=1) new_attributes.name = 'new_attributes' ############ UPDATE GRAPH TENSOR WITH NEW ATOM ATTRIBUTES ################### ### Node attribute contribution is located in every entry of graph[i,j,:] where ### there is a bond @ ij or when i = j (self) # Get atoms matrix (identity) atoms = T.identity_like(bonds) # (N_atom x N_atom) atoms.name = 'atoms_identity' # Combine bonds_or_atoms = bonds + atoms # (N_atom x N_atom) bonds_or_atoms.name = 'bonds_or_atoms' atom_indeces = T.arange( ones_vec.shape[0]) # 0 to N_atoms - 1 (indeces) atom_indeces.name = 'atom_indeces vector' ### Subtract previous node attribute contribution # Multiply each entry in bonds_or_atoms by the previous atom features for that column (old_features_to_sub, updates) = theano.scan( lambda i: T.outer(bonds_or_atoms[:, i], attributes[i, :]), sequences=T.arange(ones_vec.shape[0])) old_features_to_sub.name = 'old_features_to_sub' ### Add new node attribute contribution # Multiply each entry in bonds_or_atoms by the previous atom features for that column (new_features_to_add, updates) = theano.scan( lambda i: T.outer(bonds_or_atoms[:, i], new_attributes[i, :]), sequences=T.arange(ones_vec.shape[0])) new_features_to_add.name = 'new_features_to_add' # Update new graph new_graph = graph - old_features_to_sub + new_features_to_add new_graph.name = 'new_graph' return (new_attributes, new_graph)
def build(self, img_shape): if self.model is not None: print("PosNet has been constructed") else: img_input = Input(shape=(img_shape[0], img_shape[1], img_shape[2]), name='inputImg') x_conv = Conv2D(24, (8, 8), padding="valid", strides=(2, 2), name="conv1")(img_input) x_conv = BatchNormalization()(x_conv) x_conv = Activation('elu')(x_conv) print(x_conv.get_shape()) x_conv = Conv2D(36, (5, 5), padding="valid", strides=(2, 2), name="conv2")(x_conv) x_conv = BatchNormalization()(x_conv) x_conv = Activation('elu')(x_conv) print(x_conv.get_shape()) x_conv = Conv2D(48, (5, 5), padding="valid", strides=(2, 2), name="conv3")(x_conv) x_conv = BatchNormalization()(x_conv) x_conv = Activation('elu')(x_conv) print(x_conv.get_shape()) x_conv = Conv2D(64, (5, 5), padding="valid", name="conv4")(x_conv) x_conv = BatchNormalization()(x_conv) x_conv = Activation('elu')(x_conv) print(x_conv.get_shape()) x_conv = Conv2D(64, (5, 5), padding="valid", name="conv5")(x_conv) x_conv = BatchNormalization()(x_conv) x_conv = Activation('elu')(x_conv) print(x_conv.get_shape()) x_out = Flatten()(x_conv) print(x_out.get_shape()) # Cut for transfer learning is here: speed_input = Input(shape=(1, ), name='inputSpeed') x_out = Lambda(lambda x: K.concatenate(x, axis=1))( [x_out, speed_input]) x_out = Dense(200)(x_out) x_out = BatchNormalization()(x_out) x_out = Activation('elu')(x_out) x_out = Dense(200)(x_out) x_out = BatchNormalization()(x_out) x_end = Activation('elu')(x_out) # Branching from X_END to three branches (steer, throttle, position) steer = Dense(100)(x_end) steer = BatchNormalization()(steer) steer = Activation('elu')(steer) steer = Dropout(.2)(steer) steer = Dense(30)(steer) steer = BatchNormalization()(steer) steer = Activation('elu')(steer) steer = Dense(1, activation='sigmoid')(steer) steer = Lambda(lambda x: x * 10 - 5, name='outputSteer')(steer) throttle = Dense(100, name='thr1')(x_end) throttle = BatchNormalization(name='thr2')(throttle) throttle = Activation('elu')(throttle) throttle = Dropout(.2)(throttle) throttle = Dense(30, name='thr3')(throttle) throttle = BatchNormalization(name='thr4')(throttle) throttle = Activation('elu')(throttle) throttle = Dense(1, activation='sigmoid', name='thr5')(throttle) throttle = Lambda(lambda x: x * 2 - 1, name='outputThr')(throttle) position = Dropout(.3)(x_end) position = Dense(1, activation='sigmoid', name='pos5')(position) position = Lambda(lambda x: x * 2 - 1, name='outputPos')(position) self.model = Model((img_input, speed_input), (steer, throttle, position))
def call(self, inputs): channel_axis = 1 if self.data_format == 'channels_first' else -1 input_dim = K.shape(inputs)[channel_axis] // 4 index2 = self.filters * 2 index3 = self.filters * 3 if self.rank == 1: f_r = self.kernel[:, :, :self.filters] f_i = self.kernel[:, :, self.filters:index2] f_j = self.kernel[:, :, index2:index3] f_k = self.kernel[:, :, index3:] elif self.rank == 2: f_r = self.kernel[:, :, :, :self.filters] f_i = self.kernel[:, :, :, self.filters:index2] f_j = self.kernel[:, :, :, index2:index3] f_k = self.kernel[:, :, :, index3:] elif self.rank == 3: f_r = self.kernel[:, :, :, :, :self.filters] f_i = self.kernel[:, :, :, :, self.filters:index2] f_j = self.kernel[:, :, :, :, index2:index3] f_k = self.kernel[:, :, :, :, index3:] convArgs = { "strides": self.strides[0] if self.rank == 1 else self.strides, "padding": self.padding, "data_format": self.data_format, "dilation_rate": self.dilation_rate[0] if self.rank == 1 else self.dilation_rate } convFunc = {1: K.conv1d, 2: K.conv2d, 3: K.conv3d}[self.rank] # # Performing quaternion convolution # f_r._keras_shape = self.kernel_shape f_i._keras_shape = self.kernel_shape f_j._keras_shape = self.kernel_shape f_k._keras_shape = self.kernel_shape cat_kernels_4_r = K.concatenate([f_r, -f_i, -f_j, -f_k], axis=-2) cat_kernels_4_i = K.concatenate([f_i, f_r, -f_k, f_j], axis=-2) cat_kernels_4_j = K.concatenate([f_j, f_k, f_r, -f_i], axis=-2) cat_kernels_4_k = K.concatenate([f_k, -f_j, f_i, f_r], axis=-2) cat_kernels_4_quaternion = K.concatenate([ cat_kernels_4_r, cat_kernels_4_i, cat_kernels_4_j, cat_kernels_4_k ], axis=-1) cat_kernels_4_quaternion._keras_shape = self.kernel_size + ( 4 * input_dim, 4 * self.filters) output = convFunc(inputs, cat_kernels_4_quaternion, **convArgs) if self.use_bias: output = K.bias_add(output, self.bias, data_format=self.data_format) if self.activation is not None: output = self.activation(output) return output
def conductance_model(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=False): reg_val = 1 v_leak = 0 v_exc = 60 v_inh = -30 g_leak = 1 filter_shape[1] = 1 pad_x = int((filter_shape[1] - 1))+2 pad_t = int((filter_shape[0] - 1)) # Define the input as a tensor with shape input_shape image_in = Input(input_shape) s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in) s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in) s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in) g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s1) g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s2) g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s3) if fit_reversal: expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1)) squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1)) g1 = expand_last(g1) g2 = expand_last(g2) g3 = expand_last(g3) numerator_in = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1, g2, g3]) numerator = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False)(numerator_in) denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3]) vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator]) vm = squeeze_last(vm) else: g1_v_inh = Lambda(lambda lam: lam * v_inh)(g1) g2_v_exc = Lambda(lambda lam: lam * v_exc)(g2) g3_v_inh = Lambda(lambda lam: lam * v_inh)(g3) numerator = Lambda(lambda inputs: inputs[0] + inputs[1] + inputs[2])([g1_v_inh, g2_v_exc, g3_v_inh]) denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3]) vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator]) vm_bias = BiasLayer()(vm) vm_rect = Lambda(lambda lam: K.relu(lam))(vm_bias) if sum_over_space: conv_x_size = vm.get_shape().as_list()[2] else: conv_x_size = 1 combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False)(vm_rect) # Create model model = Model(inputs=image_in, outputs=combine_filters, name='conductance_model') return model, pad_x, pad_t
def max_min_mean_operator(x5d): max = K.max(x5d, axis=1, keepdims=True) min = K.min(x5d, axis=1, keepdims=True) mean = K.mean(x5d, axis=1, keepdims=True) return K.concatenate([max, min, mean], axis=1)
def shift_right(x, offset=1): assert offset > 0 return K.concatenate([K.zeros_like(x[:, :offset]), x[:, :-offset]], axis=1)
def step(self, a, states): # 重载rnn中的step r_tm1 = states[:self.nb_layers] # 读取输入的R、E、C(上时刻状态) c_tm1 = states[self.nb_layers:2*self.nb_layers] e_tm1 = states[2*self.nb_layers:3*self.nb_layers] if self.extrap_start_time is not None: t = states[-1] a = K.switch(t >= self.t_extrap, states[-2], a) # if past self.extrap_start_time, the previous prediction will be treated as the actual c = [] r = [] e = [] # R Unit for l in reversed(range(self.nb_layers)): # 由于R的计算需要前时刻和高一层的R,因此需要由上向下进行计算 inputs = [r_tm1[l], e_tm1[l]] if l < self.nb_layers - 1: inputs.append(r_up) # 除了最高层,前面的输入都是R_t-1,R_l+1,E,以及隐含的状态C # 标准LSTM过程 inputs = K.concatenate(inputs, axis=self.channel_axis) # 把各个特征图放到一起 i = self.conv_layers['i'][l].call(inputs) # 按照相应的卷积门尺寸卷积 f = self.conv_layers['f'][l].call(inputs) o = self.conv_layers['o'][l].call(inputs) _c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs) # c_t = f*c_t-1 + i*tanh(inputs) _r = o * self.LSTM_activation(_c) # r_t = o*tanh(c_t) c.insert(0, _c) r.insert(0, _r) if l > 0: r_up = self.upsample.call(_r) # 上采样 for l in range(self.nb_layers): ahat = self.conv_layers['ahat'][l].call(r[l]) # Ahat是R的卷积 if l == 0: ahat = K.minimum(ahat, self.pixel_max) # 第一层,Ahat限幅,准备作为输出图像 frame_prediction = ahat # 当output_mode == 'prediction'时输出 # compute errors e_up = self.error_activation(ahat - a) e_down = self.error_activation(a - ahat) e.append(K.concatenate((e_up, e_down), axis=self.channel_axis)) if self.output_layer_num == l: if self.output_layer_type == 'A': output = a elif self.output_layer_type == 'Ahat': output = ahat elif self.output_layer_type == 'R': output = r[l] elif self.output_layer_type == 'E': output = e[l] if l < self.nb_layers - 1: a = self.conv_layers['a'][l].call(e[l]) a = self.pool.call(a) # target for next layer if self.output_layer_type is None: if self.output_mode == 'prediction': output = frame_prediction else: for l in range(self.nb_layers): layer_error = K.mean(K.batch_flatten(e[l]), axis=-1, keepdims=True) # 各层平均误差,每层一个数 all_error = layer_error if l == 0 else K.concatenate((all_error, layer_error), axis=-1) if self.output_mode == 'error': output = all_error else: output = K.concatenate((K.batch_flatten(frame_prediction), all_error), axis=-1) states = r + c + e if self.extrap_start_time is not None: states += [frame_prediction, t + 1] return output, states
def call(self, inputs): inputs -= K.mean(inputs, axis=1, keepdims=True) inputs = K.l2_normalize(inputs, axis=1) pos = K.relu(inputs) neg = K.relu((- inputs)) return K.concatenate([pos, neg], axis=1)
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False): '''Return yolo_loss tensor Parameters ---------- yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body y_true: list of array, the output of preprocess_true_boxes anchors: array, shape=(N, 2), wh num_classes: integer ignore_thresh: float, the iou threshold whether to ignore object confidence loss Returns ------- loss: tensor, shape=(1,) ''' num_layers = len(anchors) // 3 # default setting yolo_outputs = args[:num_layers] y_true = args[num_layers:] anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2] ] if num_layers == 3 else [[3, 4, 5], [0, 1, 2]] input_shape = K.cast(K.shape(yolo_outputs[0])[1:3] * 8, K.dtype(y_true[0])) grid_shapes = [ K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers) ] loss = 0 m = K.shape(yolo_outputs[0])[0] # batch size, tensor mf = K.cast(m, K.dtype(yolo_outputs[0])) for l in range(num_layers): object_mask = y_true[l][..., 4:5] true_class_probs = y_true[l][..., 5:] grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True) pred_box = K.concatenate([pred_xy, pred_wh]) # Darknet raw box to calculate loss. raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1]) raw_true_wh = K.switch(object_mask, raw_true_wh, K.zeros_like(raw_true_wh)) # avoid log(0)=-inf box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4] # Find ignore mask, iterate over each of batch. ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True) object_mask_bool = K.cast(object_mask, 'bool') def loop_body(b, ignore_mask): true_box = tf.boolean_mask(y_true[l][b, ..., 0:4], object_mask_bool[b, ..., 0]) iou = box_iou(pred_box[b], true_box) best_iou = K.max(iou, axis=-1) ignore_mask = ignore_mask.write( b, K.cast(best_iou < ignore_thresh, K.dtype(true_box))) return b + 1, ignore_mask _, ignore_mask = K.control_flow_ops.while_loop(lambda b, *args: b < m, loop_body, [0, ignore_mask]) ignore_mask = ignore_mask.stack() ignore_mask = K.expand_dims(ignore_mask, -1) # K.binary_crossentropy is helpful to avoid exp overflow. xy_loss = object_mask * box_loss_scale * K.binary_crossentropy( raw_true_xy, raw_pred[..., 0:2], from_logits=True) wh_loss = object_mask * box_loss_scale * 0.5 * K.square( raw_true_wh - raw_pred[..., 2:4]) confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \ (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask class_loss = object_mask * K.binary_crossentropy( true_class_probs, raw_pred[..., 5:], from_logits=True) xy_loss = K.sum(xy_loss) / mf wh_loss = K.sum(wh_loss) / mf confidence_loss = K.sum(confidence_loss) / mf class_loss = K.sum(class_loss) / mf loss += xy_loss + wh_loss + confidence_loss + class_loss if print_loss: loss = tf.Print(loss, [ loss, xy_loss, wh_loss, confidence_loss, class_loss, K.sum(ignore_mask) ], message='loss: ') return loss
def call(self, inputs): if isinstance(inputs, list): x, x_mask = inputs else: x, x_mask = inputs, None seq_dim = K.int_shape(x)[-1] # 补足长度,保证可以reshape seq_len = K.shape(x)[1] pad_len = self.rate - seq_len % self.rate x = K.temporal_padding(x, (0, pad_len)) if x_mask is not None: x_mask = K.temporal_padding(x_mask, (0, pad_len)) new_seq_len = K.shape(x)[1] x = K.reshape(x, (-1, new_seq_len, seq_dim)) # 经过padding后shape可能变为None,所以重新声明一下shape # 线性变换 qw = self.reuse(self.q_dense, x) kw = self.reuse(self.k_dense, x) vw = self.reuse(self.v_dense, x) # 提取局部特征 kernel_size = 1 + 2 * self.neighbors kwp = extract_seq_patches( kw, kernel_size, self.rate) # shape=[None, seq_len, kernel_size, out_dim] vwp = extract_seq_patches( vw, kernel_size, self.rate) # shape=[None, seq_len, kernel_size, out_dim] if x_mask is not None: xp_mask = extract_seq_patches(x_mask, kernel_size, self.rate) # 形状变换 qw = K.reshape(qw, (-1, new_seq_len // self.rate, self.rate, self.heads, self.key_size)) kw = K.reshape(kw, (-1, new_seq_len // self.rate, self.rate, self.heads, self.key_size)) vw = K.reshape(vw, (-1, new_seq_len // self.rate, self.rate, self.heads, self.size_per_head)) kwp = K.reshape(kwp, (-1, new_seq_len // self.rate, self.rate, kernel_size, self.heads, self.key_size)) vwp = K.reshape(vwp, (-1, new_seq_len // self.rate, self.rate, kernel_size, self.heads, self.size_per_head)) if x_mask is not None: x_mask = K.reshape(x_mask, (-1, new_seq_len // self.rate, self.rate, 1, 1)) xp_mask = K.reshape( xp_mask, (-1, new_seq_len // self.rate, self.rate, kernel_size, 1, 1)) # 维度置换 qw = K.permute_dimensions( qw, (0, 3, 2, 1, 4)) # shape=[None, heads, r, seq_len // r, size] kw = K.permute_dimensions(kw, (0, 3, 2, 1, 4)) vw = K.permute_dimensions(vw, (0, 3, 2, 1, 4)) qwp = K.expand_dims(qw, 4) kwp = K.permute_dimensions( kwp, (0, 4, 2, 1, 3, 5)) # shape=[None, heads, r, seq_len // r, kernel_size, out_dim] vwp = K.permute_dimensions(vwp, (0, 4, 2, 1, 3, 5)) if x_mask is not None: x_mask = K.permute_dimensions(x_mask, (0, 3, 2, 1, 4)) xp_mask = K.permute_dimensions(xp_mask, (0, 4, 2, 1, 3, 5)) # Attention1 a = K.batch_dot(qw, kw, [4, 4]) / self.key_size**0.5 a = K.permute_dimensions(a, (0, 1, 2, 4, 3)) a = to_mask(a, x_mask, 'add') a = K.permute_dimensions(a, (0, 1, 2, 4, 3)) if self.mask_right: ones = K.ones_like(a[:1, :1, :1]) mask = (ones - K.tf.matrix_band_part(ones, -1, 0)) * 1e10 a = a - mask # Attention2 ap = K.batch_dot(qwp, kwp, [5, 5]) / self.key_size**0.5 ap = K.permute_dimensions(ap, (0, 1, 2, 3, 5, 4)) if x_mask is not None: ap = to_mask(ap, xp_mask, 'add') ap = K.permute_dimensions(ap, (0, 1, 2, 3, 5, 4)) if self.mask_right: mask = np.ones((1, kernel_size)) mask[:, -self.neighbors:] = 0 mask = (1 - K.constant(mask)) * 1e10 for _ in range(4): mask = K.expand_dims(mask, 0) ap = ap - mask ap = ap[..., 0, :] # 合并两个Attention A = K.concatenate([a, ap], -1) A = K.softmax(A) a, ap = A[..., :K.shape(a)[-1]], A[..., K.shape(a)[-1]:] # 完成输出1 o1 = K.batch_dot(a, vw, [4, 3]) # 完成输出2 ap = K.expand_dims(ap, -2) o2 = K.batch_dot(ap, vwp, [5, 4]) o2 = o2[..., 0, :] # 完成输出 o = o1 + o2 o = to_mask(o, x_mask, 'mul') o = K.permute_dimensions(o, (0, 3, 2, 1, 4)) o = K.reshape(o, (-1, new_seq_len, self.out_dim)) o = o[:, :-pad_len] return o
def call(self, inputs, mask=None): # premise_length = hypothesis_length in the following lines, but the names are kept separate to keep # track of the axes being normalized. # The inputs can be a two different tensors, or a concatenation. Hence, the conditional below. if isinstance(inputs, list) or isinstance(inputs, tuple): premise_embedding, hypothesis_embedding = inputs # (batch_size, premise_length), (batch_size, hypothesis_length) premise_mask, hypothesis_mask = mask else: premise_embedding = inputs[:, :self.premise_length, :] hypothesis_embedding = inputs[:, self.premise_length:, :] # (batch_size, premise_length), (batch_size, hypothesis_length) premise_mask = None if mask is None else mask[:, :self. premise_length] hypothesis_mask = None if mask is None else mask[:, self. premise_length:] if premise_mask is not None: premise_embedding = switch(K.expand_dims(premise_mask), premise_embedding, K.zeros_like(premise_embedding)) if hypothesis_mask is not None: hypothesis_embedding = switch(K.expand_dims(hypothesis_mask), hypothesis_embedding, K.zeros_like(hypothesis_embedding)) activation = activations.get(self.hidden_layer_activation) # (batch_size, premise_length, hidden_dim) projected_premise = apply_feed_forward(premise_embedding, self.attend_weights, activation) # (batch_size, hypothesis_length, hidden_dim) projected_hypothesis = apply_feed_forward(hypothesis_embedding, self.attend_weights, activation) ## Step 1: Attend p2h_alignment = self._align(projected_premise, projected_hypothesis, premise_mask, hypothesis_mask) # beta in the paper (equation 2) # (batch_size, premise_length, emb_dim) p2h_attention = self._attend(hypothesis_embedding, p2h_alignment, self.premise_length) h2p_alignment = self._align(projected_hypothesis, projected_premise, hypothesis_mask, premise_mask) # alpha in the paper (equation 2) # (batch_size, hyp_length, emb_dim) h2p_attention = self._attend(premise_embedding, h2p_alignment, self.hypothesis_length) ## Step 2: Compare # Equation 3 in the paper. compared_premise = self._compare(premise_embedding, p2h_attention) compared_hypothesis = self._compare(hypothesis_embedding, h2p_attention) ## Step 3: Aggregate # Equations 4 and 5. # (batch_size, hidden_dim * 2) aggregated_input = K.concatenate([ K.sum(compared_premise, axis=1), K.sum(compared_hypothesis, axis=1) ]) # (batch_size, hidden_dim) input_to_scorer = apply_feed_forward(aggregated_input, self.aggregate_weights, activation) # (batch_size, 2) final_activation = activations.get(self.final_activation) scores = final_activation(K.dot(input_to_scorer, self.scorer)) return scores
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_dim_capsule] # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule] inputs_expand = K.expand_dims(inputs, 1) # Replicate num_capsule dimension to prepare being multiplied by W # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule] inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) # Compute `inputs * W` by scanning inputs_tiled on dimension 0. # x.shape=[num_capsule, input_num_capsule, input_dim_capsule] # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule] # Regard the first two dimensions as `batch` dimension, # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule]. # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule] inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) """ # Begin: routing algorithm V1, dynamic ------------------------------------------------------------# # The prior for coupling coefficient, initialized as zeros. b = K.zeros(shape=[self.batch_size, self.num_capsule, self.input_num_capsule]) def body(i, b, outputs): c = tf.nn.softmax(b, dim=1) # dim=2 is the num_capsule dimension outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) if i != 1: b = b + K.batch_dot(outputs, inputs_hat, [2, 3]) return [i-1, b, outputs] cond = lambda i, b, inputs_hat: i > 0 loop_vars = [K.constant(self.num_routing), b, K.sum(inputs_hat, 2, keepdims=False)] shape_invariants = [tf.TensorShape([]), tf.TensorShape([None, self.num_capsule, self.input_num_capsule]), tf.TensorShape([None, self.num_capsule, self.dim_capsule])] _, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants) # End: routing algorithm V1, dynamic ------------------------------------------------------------# """ # Begin: Routing algorithm ---------------------------------------------------------------------# # In forward pass, `inputs_hat_stopped` = `inputs_hat`; # In backward, no gradient can flow from `inputs_hat_stopped` back to `inputs_hat`. inputs_hat_stopped = K.stop_gradient(inputs_hat) # The prior for coupling coefficient, initialized as zeros. # b.shape = [None, self.num_capsule, self.input_num_capsule]. b = tf.zeros(shape=[ K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule ]) #assert self.num_routing > 0, 'The num_routing should be > 0.' for i in range(self.num_routing): # c.shape=[batch_size, num_capsule, input_num_capsule] c = tf.nn.softmax(b, dim=1) # At last iteration, use `inputs_hat` to compute `outputs` in order to backpropagate gradient if i == self.num_routing - 1: # c.shape = [batch_size, num_capsule, input_num_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule]. # outputs.shape=[None, num_capsule, dim_capsule] outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) # [None, 10, 16] else: # Otherwise, use `inputs_hat_stopped` to update `b`. No gradients flow on this path. outputs = squash(K.batch_dot(c, inputs_hat_stopped, [2, 2])) # outputs.shape = [None, num_capsule, dim_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule]. # b.shape=[batch_size, num_capsule, input_num_capsule] b += K.batch_dot(outputs, inputs_hat_stopped, [2, 3]) # End: Routing algorithm -----------------------------------------------------------------------# all = K.concatenate([outputs, c]) return all
def main(): width, height = load_img(target_image_path).size global img_height, img_width img_height = 200 img_width = int(width * img_height / height) target_image = K.constant(preprocess_image(target_image_path)) style_reference_image = K.constant(preprocess_image(style_reference_path)) combination_image = K.placeholder((1, img_height, img_width, 3)) input_tensor = K.concatenate( [target_image, style_reference_image, combination_image], axis=0) model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) output_dict = dict([(layer.name, layer.output) for layer in model.layers]) content_layer = 'block5_conv2' style_layers = [ 'block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1', 'block5_conv1' ] total_variation_weight = 1e-4 style_weight = 1. content_weight = 0.025 loss = K.variable(0.) layer_features = output_dict[content_layer] target_image_features = layer_features[0, :, :, :] combination_features = layer_features[2, :, :, :] loss = loss + content_weight * content_loss(target_image_features, combination_features) for layer_name in style_layers: layer_features = output_dict[layer_name] style_reference_features = layer_features[1, :, :, :] combination_features = layer_features[2, :, :, :] s1 = style_loss(style_reference_features, combination_features) loss = loss + (style_weight / len(style_layers)) * s1 loss = loss + total_variation_weight * total_variation_loss( combination_image) grads = K.gradients(loss, combination_image)[0] fetch_loss_and_grads = K.function([combination_image], [loss, grads]) class Evaluator(object): def __init__(self): self.loss_value = None self.grads_values = None def loss(self, x): assert self.loss_value is None x = x.reshape((1, img_height, img_width, 3)) outs = fetch_loss_and_grads([x]) loss_value = outs[0] grad_values = outs[1].flatten().astype('float64') self.loss_value = loss_value self.grad_values = grad_values return self.loss_value def grads(self, x): assert self.loss_value is not None grad_values = np.copy(self.grad_values) self.loss_value = None self.grad_values = None return grad_values evaluator = Evaluator() iterations = iter_size x = preprocess_image(target_image_path) x = x.flatten() for i in range(iterations): x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x, fprime=evaluator.grads, maxfun=20) img = x.copy().reshape((img_height, img_width, 3)) img = deprocess_image(img) percent = (int)(100.0 * i / iterations) sys.stdout.write("\r{0}{1}{2}{3}{4}".format( "\r[%2d%%]" % percent, "[", "=" * int(percent / 5), " " * (20 - int(percent / 5)), "]")) sys.stdout.flush() if (i == iterations - 1): fname = 'NST.png' save_img(fname, img) print('\n\rImage saved as %s \n\r' % fname) print('\n\r')
return image_array[:, :, :, ::-1] conv_style = convert_to_bgr(style_array) conv_content = convert_to_bgr(content_array) # Feeding images as variable into Keras style_var = backend.variable(conv_style) content_var = backend.variable(conv_content) combination_img = backend.placeholder((1, h, w, 3)) # Since using Tensorflow backend and it needs tensors, we convert the image data into one concatened tensor. input_tensor = backend.concatenate([content_var, style_var, combination_img], axis=0) # Getting the model ready model = VGG16(input_tensor=input_tensor, weights="imagenet", include_top=False) # Let's load the layers layers = dict([(layer.name, layer.output) for layer in model.layers]) layers # arbitrary values content_weight = 0.025 style_weight = 5.0 total_variation_weight = 1.0