def yolo_head(feats, anchors, num_classes, input_shape): """Convert final layer features to bounding box parameters.""" num_anchors = len(anchors) # Reshape to batch, height, width, num_anchors, box_params. anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], 1, 1]) grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, 1, 1]) grid = K.concatenate([grid_x, grid_y]) grid = K.cast(grid, K.dtype(feats)) feats = K.reshape( feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5]) box_xy = K.sigmoid(feats[..., :2]) box_wh = K.exp(feats[..., 2:4]) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:]) # Adjust preditions to each spatial grid point and anchor size. box_xy = (box_xy + grid) / K.cast(grid_shape[::-1], K.dtype(feats)) box_wh = box_wh * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats)) return box_xy, box_wh, box_confidence, box_class_probs
def vgg_from_t7(t7_file, target_layer=None): '''Extract VGG layers from a Torch .t7 model into a Keras model e.g. vgg = vgg_from_t7('vgg_normalised.t7', target_layer='relu4_1') Adapted from https://github.com/jonrei/tf-AdaIN/blob/master/AdaIN.py Converted caffe->t7 from https://github.com/xunhuang1995/AdaIN-style ''' t7 = torchfile.load(t7_file, force_8bytes_long=True) inp = Input(shape=(None, None, 3), name='vgg_input') x = inp for idx,module in enumerate(t7.modules): name = module.name.decode() if module.name is not None else None if idx == 0: name = 'preprocess' # VGG 1st layer preprocesses with a 1x1 conv to multiply by 255 and subtract BGR mean as bias if module._typename == b'nn.SpatialReflectionPadding': x = Lambda(pad_reflect)(x) elif module._typename == b'nn.SpatialConvolution': filters = module.nOutputPlane kernel_size = module.kH weight = module.weight.transpose([2,3,1,0]) bias = module.bias x = Conv2D(filters, kernel_size, padding='valid', activation=None, name=name, kernel_initializer=lambda shape: K.constant(weight, shape=shape), bias_initializer=lambda shape: K.constant(bias, shape=shape), trainable=False)(x) elif module._typename == b'nn.ReLU': x = Activation('relu', name=name)(x) elif module._typename == b'nn.SpatialMaxPooling': x = MaxPooling2D(padding='same', name=name)(x) # elif module._typename == b'nn.SpatialUpSamplingNearest': # Not needed for VGG # x = Upsampling2D(name=name)(x) else: raise NotImplementedError(module._typename) if name == target_layer: # print("Reached target layer", target_layer) break # Hook it up model = Model(inputs=inp, outputs=x) return model
def call(self, x, mask=None): if mask is not None: mask = K.cast(mask, K.floatx()) mask = K.expand_dims(mask, axis=-1) s = K.sum(mask, axis=1) if K.equal(s, K.zeros_like(s)) is None: return K.mean(x, axis=1) else: return K.cast(K.sum(x * mask, axis=1) / (K.sqrt(s) + K.constant(1e-10, dtype=K.floatx())), K.floatx()) else: print (x) return K.mean(x, axis=1)
def softmax(x, axis, mask=None): if mask is None: mask = K.constant(True) mask = K.cast(mask, K.floatx()) if K.ndim(x) is K.ndim(mask) + 1: mask = K.expand_dims(mask) m = K.max(x, axis=axis, keepdims=True) e = K.exp(x - m) * mask s = K.sum(e, axis=axis, keepdims=True) s += K.cast(K.cast(s < K.epsilon(), K.floatx()) * K.epsilon(), K.floatx()) return e / s
def result(input): emb_c = emb_layer(input) conv_results_list = [] for cl in conv2d_layers: conv_results_list.append(cl(emb_c)) emb_c = Lambda(lambda x: K.concatenate(x, axis=3))(conv_results_list) emb_c = Lambda(lambda x: K.max(x, axis=2))(emb_c) if highway_on_top: sigmoid_gate = dense1(emb_c) sigmoid_gate = Activation('sigmoid')(sigmoid_gate) deeper_units = dense2(emb_c) emb_c = Add()([Multiply()([sigmoid_gate, deeper_units]), Multiply()([Lambda(lambda x: K.constant(1., shape=K.shape(x)) - x)(sigmoid_gate), emb_c])]) emb_c = Activation('relu')(emb_c) return emb_c
def eval(outputs, anchors, num_classes, image_shape, max_boxes=20, score_threshold=.6, iou_threshold=.5): '''Evaluate the YOLO model on given input and return filtered boxes''' num_layers = len(outputs) anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [ [3, 4, 5], [1, 2, 3]] input_shape = K.shape(outputs[0])[1:3] * 32 boxes = [] box_scores = [] for l in range(num_layers): _boxes, _box_scores = boxes_and_scores(outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) mask = box_scores >= score_threshold max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] for c in range(num_classes): # TODO: use Keras backend instead of tf. class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) nms_index = tf.image.non_max_suppression( class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_
def call(self, inputs, **kwargs): input_shape = K.int_shape(inputs) broadcast_shape = [1] * len(input_shape) broadcast_shape[self.axis] = input_shape[self.axis] broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape) broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape) broadcast_gamma = K.reshape(self.gamma, broadcast_shape) broadcast_beta = K.reshape(self.beta, broadcast_shape) invstd = (K.ones(shape=broadcast_shape, dtype='float32') / K.sqrt(broadcast_moving_variance + K.constant(self.epsilon, dtype='float32'))) return((inputs - broadcast_moving_mean) * invstd * broadcast_gamma + broadcast_beta)
def weighted_binary_crossentropy(y_true, y_pred): class_loglosses = K.mean(K.binary_crossentropy(y_true, y_pred), axis=[0, 1, 2]) return K.sum(class_loglosses * K.constant(class_weights))
num_nodes_min_max_tr, theta, True) A = [] Y = [] for index in range(len(input_graphs)): A.append(networkx.convert_matrix.to_numpy_array(input_graphs[index])) Y.append(networkx.convert_matrix.to_numpy_array(target_graphs[index])) print(A) print(Y) SYM_NORM = True A_norm = examples.utils.preprocess_adj_numpy(A, SYM_NORM) num_filters = 2 graph_conv_filters = examples.utils.np.concatenate( [A_norm, examples.utils.np.matmul(A_norm, A_norm)], axis=0) graph_conv_filters = K.constant(graph_conv_filters) model = Sequential() model.add( GraphCNN(16, num_filters, graph_conv_filters, input_shape=(X.shape[1], ), activation='elu', kernel_regularizer=l2(5e-4))) model.add(Dropout(0.2)) model.add( GraphCNN(Y.shape[1], num_filters, graph_conv_filters, activation='elu',
def log_normals_loss(y_true, y_pred): y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30) y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30) #compute normals with convolution approach # (http://answers.opencv.org/question/82453/calculate-surface-normals-from-depth-image-using-neighboring-pixels-cross-product/) config, unparsed = get_config() batch_size = config.batch_size #y_true_clipped = K.clip(y_true, K.epsilon(), None)#40.0) #y_pred_clipped = K.clip(y_pred, K.epsilon(), None) y_true_clipped = y_true y_pred_clipped = y_pred filter_y = K.variable(np.array([[ 0., -0.5 , 0.], [0., 0., 0.], [0., 0.5, 0.]]).reshape(3, 3, 1, 1)) filter_x = K.variable(np.array([ [0, 0., 0.], [0.5, 0., -0.5], [0., 0., 0.]]).reshape(3, 3, 1, 1)) w_x = K.variable(np.array([[-1., 0., 1.], [-1., 0., 1.], [-1., 0., 1.]]).reshape(3, 3, 1, 1)) w_y = K.variable(np.array([[-1., -1., -1.], [0., 0., 0.], [1., 1., 1.]]).reshape(3, 3, 1, 1)) #dzdx = K.conv2d(K.exp(y_true_clipped), w_x, padding='same') #dzdy = K.conv2d(K.exp(y_true_clipped), w_y, padding='same') dzdx = K.conv2d(y_true_clipped, w_x, padding='same') dzdy = K.conv2d(y_true_clipped, w_y, padding='same') dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx)) dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy)) mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx)) mag_norm = K.sqrt(mag_norm) N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm N1 = dzdx_ / mag_norm N2 = dzdy_ / mag_norm normals = K.concatenate(tensors=[N1,N2,N3],axis=-1) #dzdx_pred = K.conv2d(K.exp(y_pred_clipped), w_x, padding='same') #dzdy_pred = K.conv2d(K.exp(y_pred_clipped), w_y, padding='same') dzdx_pred = K.conv2d(y_pred_clipped, w_x, padding='same') dzdy_pred = K.conv2d(y_pred_clipped, w_y, padding='same') mag_norm_pred_x = K.pow(dzdx_pred,2) + 1.0 mag_norm_pred_x = K.sqrt(mag_norm_pred_x) mag_norm_pred_y = K.pow(dzdy_pred, 2) + 1.0 mag_norm_pred_y = K.sqrt(mag_norm_pred_y) grad_x = K.concatenate(tensors=[K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x, K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x, dzdx_pred/ mag_norm_pred_x],axis=-1) grad_y = K.concatenate(tensors=[K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y, K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y, dzdy_pred/ mag_norm_pred_y],axis=-1) first_log = K.log(y_pred_clipped + 1.) second_log = K.log(y_true_clipped + 1.) log_term = K.sqrt(K.mean(K.square(first_log - second_log), axis=-1) + 0.00001) dot_term_x = K.sum(normals[:,:,:,:] * grad_x[:,:,:,:], axis=-1, keepdims=True) dot_term_y = K.sum(normals[:,:,:,:] * grad_y[:,:,:,:], axis=-1, keepdims=True) #dot_term_x = K.mean(K.sum(normals[:, :, :, :] * grad_x[:, :, :, :], axis=-1, keepdims=True), axis=-1) #dot_term_y = K.mean(K.sum(normals[:, :, :, :] * grad_y[:, :, :, :], axis=-1, keepdims=True), axis=-1) dot_term_x = tf.Print(dot_term_x, [dot_term_x], message='dot_term_x', summarize=30) dot_term_y = tf.Print(dot_term_y, [dot_term_y], message='dot_term_y', summarize=30) #commentare per vecchia versione sc_inv_term = K.square(K.mean((first_log - second_log), axis=-1)) norm_term = K.mean(K.square(dot_term_x), axis=-1) + K.mean(K.square(dot_term_y), axis=-1) diff_x = dzdx_pred - dzdx diff_y = dzdy_pred - dzdy grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1) loss = log_term - (0.5 * sc_inv_term) + norm_term #+ grad_loss #loss = log_term + K.square(dot_term_x) + K.square(dot_term_y) return loss
def custom_mod(x, k=K.constant(np.pi), name="custom_mod"): """ Custom mod function """ y = Lambda(lambda x: K.tf.math.floormod(x - k, 2*k) - k, name=name)(x) return y
def ccc_loss(gold, pred): # input (num_batches, seq_len, 1) ccc_loss = K.constant(1.) - ccc(gold, pred) return ccc_loss
def call(self, x, mask=None): ''' Return an anchor box tensor based on the shape of the input tensor. The logic implemented here is identical to the logic in the module `ssd_box_encode_decode_utils.py`. Note that this tensor does not participate in any graph computations at runtime. It is being created as a constant once during graph creation and is just being output along with the rest of the model output during runtime. Because of this, all logic is implemented as Numpy array operations and it is sufficient to convert the resulting Numpy array into a Keras tensor at the very end before outputting it. Arguments: x (tensor): 4D tensor of shape `(batch, channels, height, width)` if `dim_ordering = 'th'` or `(batch, height, width, channels)` if `dim_ordering = 'tf'`. The input for this layer must be the output of the localization predictor layer. ''' # Compute box width and height for each aspect ratio # The shorter side of the image will be used to compute `w` and `h` using `scale` and `aspect_ratios`. self.aspect_ratios = np.sort(self.aspect_ratios) size = min(self.img_height, self.img_width) # Compute the box widths and and heights for all aspect ratios wh_list = [] for ar in self.aspect_ratios: if (ar == 1) & self.two_boxes_for_ar1: # Compute the regular default box for aspect ratio 1 and... w = self.this_scale * size * np.sqrt(ar) h = self.this_scale * size / np.sqrt(ar) wh_list.append((w,h)) # ...also compute one slightly larger version using the geometric mean of this scale value and the next w = np.sqrt(self.this_scale * self.next_scale) * size * np.sqrt(ar) h = np.sqrt(self.this_scale * self.next_scale) * size / np.sqrt(ar) wh_list.append((w,h)) else: w = self.this_scale * size * np.sqrt(ar) h = self.this_scale * size / np.sqrt(ar) wh_list.append((w,h)) wh_list = np.array(wh_list) # We need the shape of the input tensor if K.image_dim_ordering() == 'tf': batch_size, feature_map_height, feature_map_width, feature_map_channels = x._keras_shape else: # Not yet relevant since TensorFlow is the only supported backend right now, but it can't harm to have this in here for the future batch_size, feature_map_channels, feature_map_height, feature_map_width = x._keras_shape # Compute the grid of box center points. They are identical for all aspect ratios cell_height = self.img_height / feature_map_height cell_width = self.img_width / feature_map_width cx = np.linspace(cell_width/2, self.img_width-cell_width/2, feature_map_width) cy = np.linspace(cell_height/2, self.img_height-cell_height/2, feature_map_height) cx_grid, cy_grid = np.meshgrid(cx, cy) cx_grid = np.expand_dims(cx_grid, -1) # This is necessary for np.tile() to do what we want further down cy_grid = np.expand_dims(cy_grid, -1) # This is necessary for np.tile() to do what we want further down # Create a 4D tensor template of shape `(feature_map_height, feature_map_width, n_boxes, 4)` # where the last dimension will contain `(cx, cy, w, h)` boxes_tensor = np.zeros((feature_map_height, feature_map_width, self.n_boxes, 4)) boxes_tensor[:, :, :, 0] = np.tile(cx_grid, (1, 1, self.n_boxes)) # Set cx boxes_tensor[:, :, :, 1] = np.tile(cy_grid, (1, 1, self.n_boxes)) # Set cy boxes_tensor[:, :, :, 2] = wh_list[:, 0] # Set w boxes_tensor[:, :, :, 3] = wh_list[:, 1] # Set h # Convert `(cx, cy, w, h)` to `(xmin, xmax, ymin, ymax)` boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='centroids2minmax') # If `limit_boxes` is enabled, clip the coordinates to lie within the image boundaries if self.limit_boxes: x_coords = boxes_tensor[:,:,:,[0, 1]] x_coords[x_coords >= self.img_width] = self.img_width - 1 x_coords[x_coords < 0] = 0 boxes_tensor[:,:,:,[0, 1]] = x_coords y_coords = boxes_tensor[:,:,:,[2, 3]] y_coords[y_coords >= self.img_height] = self.img_height - 1 y_coords[y_coords < 0] = 0 boxes_tensor[:,:,:,[2, 3]] = y_coords # `normalize_coords` is enabled, normalize the coordinates to be within [0,1] if self.normalize_coords: boxes_tensor[:, :, :, :2] /= self.img_width boxes_tensor[:, :, :, 2:] /= self.img_height if self.coords == 'centroids': # TODO: Implement box limiting directly for `(cx, cy, w, h)` so that we don't have to unnecessarily convert back and forth # Convert `(xmin, xmax, ymin, ymax)` back to `(cx, cy, w, h)` boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='minmax2centroids') # 4: Create a tensor to contain the variances and append it to `boxes_tensor`. This tensor has the same shape # as `boxes_tensor` and simply contains the same 4 variance values for every position in the last axis. variances_tensor = np.zeros_like(boxes_tensor) # Has shape `(feature_map_height, feature_map_width, n_boxes, 4)` variances_tensor += self.variances # Long live broadcasting # Now `boxes_tensor` becomes a tensor of shape `(feature_map_height, feature_map_width, n_boxes, 8)` boxes_tensor = np.concatenate((boxes_tensor, variances_tensor), axis=-1) # Now prepend one dimension to `boxes_tensor` to account for the batch size and tile it along # The result will be a 5D tensor of shape `(batch_size, feature_map_height, feature_map_width, n_boxes, 8)` boxes_tensor = np.expand_dims(boxes_tensor, axis=0) boxes_tensor = K.tile(K.constant(boxes_tensor, dtype='float32'), (K.shape(x)[0], 1, 1, 1, 1)) return boxes_tensor
def jaccard_loss(y_true, y_pred, weights): batch_jaccard_coefs = soft_jaccard(y_true, y_pred, axis=[1, 2]) jaccard_coefs = K.mean(batch_jaccard_coefs, axis=0) w = K.constant(weights) / sum(weights) return 1 - K.sum(w * jaccard_coefs)
import numpy as np import tensorflow.spectral as tf from keras import backend as K #from so3_fft import so3_rfft, _setup_wigner x = [[1.1, 2.2], [3.3, 4.4]] x = K.constant(x, dtype='complex64') y = K.eval(x) print(y.shape[0])
def corr(y_true, y_pred): return K.constant((numpy.corrcoef(y_true.flatten(), y_pred.flatten())[0, 1]) ** 2)
def _build_model(self): rpn_trainable = self.config.training_mode in ['rpn_only', 'all'] head_trainable = self.config.training_mode in ['head_only', 'all'] # backbone network backbone_in, backbone_out = self._model_backbone_headless() # rpn normalized_rois, rpn_offsets, objects, objects_logit \ = self._nn_rpn(backbone_out, rpn_trainable) # 学習時のみ損失を計算 if self.config.training: # 学習時 input_gt_rois = Input( shape=[None, 4], name="input_gt_rois", dtype='float32') input_gt_objects = Input( shape=[None], name="input_gt_objects", dtype='int32') inputs = [backbone_in, input_gt_rois, input_gt_objects] losses = [] if rpn_trainable: # 損失計算 # RPNの損失 rpn_offsets_loss = Lambda(lambda x: loss.rpn_offsets_loss(*x), name="rpn_offsets_loss")( [input_gt_rois, input_gt_objects, rpn_offsets]) rpn_objects_loss = Lambda(lambda x: loss.rpn_objects_loss(*x), name="rpn_objects_loss")( [input_gt_objects, objects]) losses += [rpn_offsets_loss, rpn_objects_loss] if head_trainable: input_gt_boxes = Input( shape=[None, 4], name="input_gt_boxes", dtype='float32') input_gt_label_ids = Input( shape=[None], name="input_gt_label_ids", dtype='int32') h, w = self.config.image_shape[0], self.config.image_shape[1] input_gt_masks = Input( shape=[None, h, w], name="input_gt_masks", dtype='float32') inputs += [input_gt_boxes, input_gt_label_ids, input_gt_masks] # 正解データとRoIから評価対象のRoIを絞り込み、それに対応する正解データを得る。 normalized_sample_rois, normalized_sample_gt_offsets, \ sample_gt_labels, sample_gt_masks = \ MaskSubsamplingRoiLayer(self.config, name='mask_subsampling')( [normalized_rois, input_gt_boxes, input_gt_label_ids, input_gt_masks]) # head head_offsets, labels, labels_logit \ = self._nn_head(backbone_out, normalized_sample_rois) # ヘッドの損失はModel#compileで損失関数を指定する方法では対応出来ないため、 # Layerとして定義してModel#add_lossで加算する。 head_offsets_loss = Lambda(lambda x: loss.head_offsets_loss(*x), name="head_offsets_loss")( [normalized_sample_gt_offsets, sample_gt_labels, head_offsets]) head_labels_loss = Lambda(lambda x: loss.head_labels_loss(*x), name="head_labels_loss")( [sample_gt_labels, labels]) # mask masks = self._nn_mask(backbone_out, normalized_sample_rois) head_mask_loss = Lambda(lambda x: loss.head_mask_loss(*x), name="head_mask_loss")( [sample_gt_masks, sample_gt_labels, masks]) # 損失 losses += [head_offsets_loss, head_labels_loss, head_mask_loss] # 出力=損失 outputs = losses else: # 予測時 # head # head_offsetsは0〜1で正規化された値 head_offsets, labels, _ = self._nn_head( backbone_out, normalized_rois) # 候補を絞り込む bboxes, rois, labels, scores = Lambda(lambda x: self._nn_squeeze_roi(*x), name="squeeze_roi")( [normalized_rois, head_offsets, labels]) # mask masks = self._nn_mask(backbone_out, rois) def _squeeze_masks(masks, idx_labels): dim1 = K.flatten(K.repeat(K.expand_dims( K.arange(K.shape(masks)[0])), K.shape(masks)[1])) dim2 = K.tile(K.arange(K.shape(masks)[1]), [K.shape(masks)[0]]) idx = K.stack([dim1, dim2, K.cast(K.flatten(labels), tf.int32)], axis=1) # idx = log.tfprint(idx, "idx:inloop:debug") squeezed_masks = tf.gather_nd(masks, idx) squeezed_masks = K.reshape(squeezed_masks, [K.shape(masks)[0], K.shape(masks)[1], K.shape(masks)[3], K.shape(masks)[4]]) return squeezed_masks # ラベルに対応するマスクを残す masks = Lambda(lambda x: _squeeze_masks(x[0], K.cast(x[1], tf.int32)), name="squeeze_mask")([masks, labels]) # 予測時は損失不要 # ダミーの損失関数 dummy_loss = Lambda(lambda x: K.constant(0), name="dummy_loss")( [backbone_in]) losses = [dummy_loss, dummy_loss, dummy_loss, dummy_loss, dummy_loss, dummy_loss] inputs = [backbone_in] outputs = [bboxes, labels, scores, masks, rois, rpn_offsets, objects] model = Model(inputs=inputs, outputs=outputs, name='mask_r_cnn') # Kerasは複数指定した損失の合計をモデル全体の損失として評価してくれる。 # 損失を追加 for output in losses: model.add_loss(tf.reduce_mean(output, keep_dims=True)) return model, len(outputs)
def _smooth_labels(y_true, label_smoothing): num_classes = tf.cast(K.shape(y_true)[-1], dtype=K.floatx()) label_smoothing = K.constant(label_smoothing, dtype=K.floatx()) return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes
def drop_last_row(shape, dtype=None): extractor = np.zeros(shape) for i in range(np.min(shape)): extractor[i][i] = 1 return K.constant(extractor)
def extract_last_row(shape, dtype=None): extractor = np.zeros(shape) extractor[-1][-1] = 0 return K.constant(extractor)
def __call__(self, shape, dtype=None): return K.constant(1 / K.sqrt(16), shape=shape, dtype=dtype)
def yolo_eval(yolo_outputs, anchors, num_classes, image_shape, max_boxes=20, score_threshold=.6, iou_threshold=.5): # 获得特征层的数量 num_layers = len(yolo_outputs) # 特征层1对应的anchor是678(13,13) # 特征层2对应的anchor是345(26,26) # 特征层3对应的anchor是012(52,52) anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] input_shape = K.shape(yolo_outputs[0])[1:3] * 32 # 416,416 boxes = [] box_scores = [] # ---------------------------------对每一个特征层的结果进行处理----------------------- # 对每个特征层进行处理 for l in range(num_layers): _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) # 将每个特征层的结果进行堆叠(全部堆叠成一维) boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) # 每一个点对应80种类别的得分值 # 通过每个得分和阈值进行比较,满足要求的就留下,并设置为1 mask = box_scores >= score_threshold # 限制每个图片最大的边框数(20) max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] for c in range(num_classes): # 取出所有box_scores >= score_threshold的框,和成绩 class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) # 非极大抑制,去掉box重合程度高的那一些 nms_index = tf.image.non_max_suppression(class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) # 获取非极大抑制后的结果 # 下列三个分别是 # 框的位置,得分与种类 class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_
def __init__(self, switch_matrix, pool_size): super(MaxUnPooling2D, self).__init__() self.switch_matrix = K.constant(switch_matrix) self.pool_size = pool_size
def mean_squared_error(y_true, y_pred): if not K.is_tensor(y_pred): y_pred = K.constant(y_pred) y_true = K.cast(y_true, y_pred.dtype) return K.mean(K.square(y_pred - y_true), axis=-1)
def call(self, x, mask=None): ''' Return an anchor box tensor based on the shape of the input tensor. The logic implemented here is identical to the logic in the module `ssd_box_encode_decode_utils.py`. Note that this tensor does not participate in any graph computations at runtime. It is being created as a constant once during graph creation and is just being output along with the rest of the model output during runtime. Because of this, all logic is implemented as Numpy array operations and it is sufficient to convert the resulting Numpy array into a Keras tensor at the very end before outputting it. Arguments: x (tensor): 4D tensor of shape `(batch, channels, height, width)` if `dim_ordering = 'th'` or `(batch, height, width, channels)` if `dim_ordering = 'tf'`. The input for this layer must be the output of the localization predictor layer. ''' # Compute box width and height for each aspect ratio # The shorter side of the image will be used to compute `w` and `h` using `scale` and `aspect_ratios`. size = min(self.img_height, self.img_width) # Compute the box widths and and heights for all aspect ratios wh_list = [] for ar in self.aspect_ratios: if (ar == 1): # Compute the regular anchor box for aspect ratio 1. box_height = box_width = self.this_scale * size wh_list.append((box_width, box_height)) if self.two_boxes_for_ar1: # Compute one slightly larger version using the geometric mean of this scale value and the next. box_height = box_width = np.sqrt(self.this_scale * self.next_scale) * size wh_list.append((box_width, box_height)) else: box_height = self.this_scale * size / np.sqrt(ar) box_width = self.this_scale * size * np.sqrt(ar) wh_list.append((box_width, box_height)) wh_list = np.array(wh_list) # We need the shape of the input tensor if K.image_dim_ordering() == 'tf': batch_size, feature_map_height, feature_map_width, feature_map_channels = x._keras_shape else: # Not yet relevant since TensorFlow is the only supported backend right now, but it can't harm to have this in here for the future batch_size, feature_map_channels, feature_map_height, feature_map_width = x._keras_shape # Compute the grid of box center points. They are identical for all aspect ratios. # Compute the step sizes, i.e. how far apart the anchor box center points will be vertically and horizontally. if (self.this_steps is None): step_height = self.img_height / feature_map_height step_width = self.img_width / feature_map_width else: if isinstance(self.this_steps, (list, tuple)) and (len(self.this_steps) == 2): step_height = self.this_steps[0] step_width = self.this_steps[1] elif isinstance(self.this_steps, (int, float)): step_height = self.this_steps step_width = self.this_steps # Compute the offsets, i.e. at what pixel values the first anchor box center point will be from the top and from the left of the image. if (self.this_offsets is None): offset_height = 0.5 offset_width = 0.5 else: if isinstance(self.this_offsets, (list, tuple)) and (len(self.this_offsets) == 2): offset_height = self.this_offsets[0] offset_width = self.this_offsets[1] elif isinstance(self.this_offsets, (int, float)): offset_height = self.this_offsets offset_width = self.this_offsets # Now that we have the offsets and step sizes, compute the grid of anchor box center points. cy = np.linspace(offset_height * step_height, (offset_height + feature_map_height - 1) * step_height, feature_map_height) cx = np.linspace(offset_width * step_width, (offset_width + feature_map_width - 1) * step_width, feature_map_width) cx_grid, cy_grid = np.meshgrid(cx, cy) cx_grid = np.expand_dims(cx_grid, -1) # This is necessary for np.tile() to do what we want further down cy_grid = np.expand_dims(cy_grid, -1) # This is necessary for np.tile() to do what we want further down # Create a 4D tensor template of shape `(feature_map_height, feature_map_width, n_boxes, 4)` # where the last dimension will contain `(cx, cy, w, h)` boxes_tensor = np.zeros((feature_map_height, feature_map_width, self.n_boxes, 4)) boxes_tensor[:, :, :, 0] = np.tile(cx_grid, (1, 1, self.n_boxes)) # Set cx boxes_tensor[:, :, :, 1] = np.tile(cy_grid, (1, 1, self.n_boxes)) # Set cy boxes_tensor[:, :, :, 2] = wh_list[:, 0] # Set w boxes_tensor[:, :, :, 3] = wh_list[:, 1] # Set h # Convert `(cx, cy, w, h)` to `(xmin, xmax, ymin, ymax)` boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='centroids2corners') # If `clip_boxes` is enabled, clip the coordinates to lie within the image boundaries if self.clip_boxes: x_coords = boxes_tensor[:, :, :, [0, 2]] x_coords[x_coords >= self.img_width] = self.img_width - 1 x_coords[x_coords < 0] = 0 boxes_tensor[:, :, :, [0, 2]] = x_coords y_coords = boxes_tensor[:, :, :, [1, 3]] y_coords[y_coords >= self.img_height] = self.img_height - 1 y_coords[y_coords < 0] = 0 boxes_tensor[:, :, :, [1, 3]] = y_coords # If `normalize_coords` is enabled, normalize the coordinates to be within [0,1] if self.normalize_coords: boxes_tensor[:, :, :, [0, 2]] /= self.img_width boxes_tensor[:, :, :, [1, 3]] /= self.img_height # TODO: Implement box limiting directly for `(cx, cy, w, h)` so that we don't have to unnecessarily convert back and forth. if self.coords == 'centroids': # Convert `(xmin, ymin, xmax, ymax)` back to `(cx, cy, w, h)`. boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='corners2centroids') elif self.coords == 'minmax': # Convert `(xmin, ymin, xmax, ymax)` to `(xmin, xmax, ymin, ymax). boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='corners2minmax') # Create a tensor to contain the variances and append it to `boxes_tensor`. This tensor has the same shape # as `boxes_tensor` and simply contains the same 4 variance values for every position in the last axis. variances_tensor = np.zeros_like( boxes_tensor) # Has shape `(feature_map_height, feature_map_width, n_boxes, 4)` variances_tensor += self.variances # Long live broadcasting # Now `boxes_tensor` becomes a tensor of shape `(feature_map_height, feature_map_width, n_boxes, 8)` boxes_tensor = np.concatenate((boxes_tensor, variances_tensor), axis=-1) # Now prepend one dimension to `boxes_tensor` to account for the batch size and tile it along # The result will be a 5D tensor of shape `(batch_size, feature_map_height, feature_map_width, n_boxes, 8)` boxes_tensor = np.expand_dims(boxes_tensor, axis=0) boxes_tensor = K.tile(K.constant(boxes_tensor, dtype='float32'), (K.shape(x)[0], 1, 1, 1, 1)) return boxes_tensor
def mean_absolute_error_penalty(y_true, y_pred, penalty): penalty = K.constant(penalty, dtype='float32') return K.mean(penalty * K.abs(y_pred - y_true), axis=-1)
def call(self, inputs, training=None): input_shape = K.int_shape(inputs) broadcast_shape = [1] * len(input_shape) broadcast_shape[self.axis] = input_shape[self.axis] broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape) broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape) broadcast_gamma = K.reshape(self.gamma, broadcast_shape) broadcast_beta = K.reshape(self.beta, broadcast_shape) invstd = K.ones (shape=broadcast_shape, dtype='float32') / K.sqrt(broadcast_moving_variance + K.constant(self.epsilon, dtype='float32')) return (inputs - broadcast_moving_mean) * invstd * broadcast_gamma + broadcast_beta
def fit(self, Xs, ys, dataset_weights=None): self.random_state = check_random_state(self.random_state) n_datasets = len(Xs) n_samples = sum(X.shape[0] for X in Xs) n_features = Xs[0].shape[1] # Data sizes = np.array([y.shape[1] for y in ys]) limits = [0] + np.cumsum(sizes).tolist() n_targets = limits[-1] if self.n_components == 'auto': n_components = n_targets else: n_components = self.n_components self.slices_ = [] for iter in range(n_datasets): self.slices_.append(np.array([limits[iter], limits[iter + 1]])) self.slices_ = tuple(self.slices_) if dataset_weights is None: dataset_weights = [1.] * n_datasets dataset_weights /= np.sum(dataset_weights) / n_datasets # dataset_weights = np.array(dataset_weights) * np.sqrt([X.shape[0] for X in Xs]) padded_ys = [] masks = [] Xs = [X.copy() for X in Xs] for y, this_slice in zip(ys, self.slices_): padded_y = np.zeros((y.shape[0], n_targets)) mask = np.zeros((y.shape[0], n_targets), dtype='bool') mask[:, this_slice[0]:this_slice[1]] = 1 padded_y[:, this_slice[0]:this_slice[1]] = y padded_ys.append(padded_y) masks.append(mask) if self.use_generator: our_generator = generator(Xs, padded_ys, masks, batch_size=self.batch_size, dataset_weights=dataset_weights, random_state=self.random_state) if self.batch_size is not None: steps_per_epoch = n_samples / self.batch_size else: steps_per_epoch = n_datasets else: Xs_cat = np.concatenate(Xs, axis=0) padded_ys_cat = np.concatenate(padded_ys, axis=0) masks_cat = np.concatenate(masks, axis=0) sample_weight = np.concatenate( [[dataset_weight * n_samples / X.shape[0] / n_datasets] * X.shape[0] for dataset_weight, X in zip(dataset_weights, Xs)]) if self.batch_size is None: batch_size = n_samples else: batch_size = self.batch_size # Model if self.intercept_init is not None: supervised_bias_initializer = \ lambda shape: K.constant(self.intercept_init) else: supervised_bias_initializer = 'zeros' if self.coef_init is not None: U, S, VT = svd(self.coef_init, full_matrices=False) latent_init = U[:, :n_components] latent_init *= S[:n_components] supervised_init = VT[:n_components] if n_components > latent_init.shape[1]: latent_init = np.concatenate([ latent_init, np.zeros((latent_init.shape[0], n_components - latent_init.shape[1])) ], axis=1) supervised_init = np.concatenate([ supervised_init, np.zeros((n_components - supervised_init.shape[0], supervised_init.shape[1])) ], axis=0) supervised_kernel_initializer = \ lambda shape: K.constant(supervised_init) latent_kernel_initializer = \ lambda shape: K.constant(latent_init) else: supervised_kernel_initializer = 'glorot_uniform' latent_kernel_initializer = 'glorot_uniform' data = Input(shape=(n_features, ), name='data', dtype='float32') mask = Input(shape=(n_targets, ), name='mask', dtype='bool') dropout_data = Dropout(rate=self.input_dropout_rate)(data) if n_components is not None: latent = Dense(n_components, name='latent', use_bias=False, kernel_initializer=latent_kernel_initializer, kernel_regularizer=l2(self.alpha))(dropout_data) latent = Dropout(rate=self.latent_dropout_rate)(latent) else: latent = dropout_data logits = Dense(n_targets, use_bias=True, name='supervised', kernel_initializer=supervised_kernel_initializer, bias_initializer=supervised_bias_initializer, kernel_regularizer=l2(self.alpha))(latent) prediction = PartialSoftmax()([logits, mask]) self.model_ = Model(inputs=[data, mask], outputs=prediction) if self.optimizer == 'adam': optimizer = Adam elif self.optimizer == 'sgd': optimizer = SGD elif self.optimizer == 'rmsprop': optimizer = RMSprop else: raise ValueError('Wrong optimizer') self.model_.compile(optimizer(self.step_size), loss='categorical_crossentropy', metrics=['accuracy']) schedule = lambda i: 1e-3 if i < self.max_iter // 2 else 1e-4 scheduler = LearningRateScheduler(schedule) callbacks = [scheduler] if self.latent_sparsity is not None: callbacks.append(L1ProjCallback(radius=self.latent_sparsity)) sess = tf.Session( config=tf.ConfigProto(device_count={'CPU': self.n_jobs}, inter_op_parallelism_threads=self.n_jobs, intra_op_parallelism_threads=self.n_jobs, use_per_session_threads=True)) K.set_session(sess) if self.use_generator: self.model_.fit_generator(our_generator, steps_per_epoch=steps_per_epoch, verbose=2, epochs=self.max_iter, callbacks=callbacks) else: self.model_.fit([Xs_cat, masks_cat], padded_ys_cat, sample_weight=sample_weight, batch_size=batch_size, verbose=2, epochs=self.max_iter, callbacks=callbacks) supervised, self.intercept_ = self.model_.get_layer( 'supervised').get_weights() if self.n_components is not None: latent = self.model_.get_layer('latent').get_weights()[0] self.coef_ = latent.dot(supervised) else: self.coef_ = supervised
img = vgg19.preprocess_input(img) return img def deprocess_image(x): x[:, :, 0] += 103.939 x[:, :, 1] += 116.779 x[:, :, 2] += 123.68 # BGR -> RGB x = x[:, :, ::-1] x = np.clip(x, 0, 255).astype('uint8') return x # 读取VGG19网络并应用到我们的图片中 target_image = K.constant(preprocess_image(target_image_path)) style_reference_image = K.constant(preprocess_image(style_reference_image_path)) # 生成的图像 combination_image = K.placeholder((1, img_height, img_width, 3)) input_tensor = K.concatenate([target_image, style_reference_image, combination_image], axis=0) model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) print('Model loaded.')
def __init__(self, wordvec=base + 'data/wordvec.txt', window=1, maxlen=100, filter=250, pe=False, drop_emb=.5, drop_conv=.5, drop_sa=.25, drop_res=0, drop_dense=0, feature=0, n_layers=1, sa_scale=0, residue=False, dense_layers=[], pad_position=['pre', 'pre'], n_relation=4, n_clause=2, torel=None): if type(dense_layers) is int: dense_layers = [dense_layers] self.__dict__.update(locals()) if torel is not None: self.n_relation = n_relation = len(torel) self.wordvec = wordvec = word2vec.load(wordvec) vs, es = wordvec.vectors.shape # Input Layers inputs = [ Input(shape=(maxlen, ), name='clause%d' % i) for i in range(n_clause) ] if self.feature: feature = [Input(shape=(feature, ), name='feature')] else: feature = [] # Hidden Layers embed = Embedding(vs, es, input_length=maxlen, weights=[wordvec.vectors], name='embed') if sa_scale: attn = Lambda(scaled_attention(sa_scale), name='attention') drop_sa = Dropout(drop_sa, name='drop_sa') if not hasattr(filter, '__getitem__'): filter = (filter, ) * n_layers if not hasattr(drop_conv, '__getitem__'): drop_conv = (drop_conv, ) * n_layers if not hasattr(drop_res, '__getitem__'): drop_res = (drop_res, ) * n_layers if not hasattr(drop_dense, '__getitem__'): drop_dense = (drop_dense, ) * len(dense_layers) if self.pe: dim = es pe_i, pe_p = np.meshgrid(range(dim), range(maxlen)) pe = np.sin(2 * np.pi * pe_p / maxlen * pe_i / dim) #pe = np.sin(pe_p/10000**(2*pe_i/dim)) #pe /= wordvec.vectors.max()*10 pe_const = K.constant(pe, name='pe') pe = Lambda(lambda x: x + pe_const, name='pos_enc') if residue: reduce = [[ Dense(filter[j], name='reduce%d-%d' % (i, j)) for j in range(n_layers) ] for i in range(n_clause)] add = Add(name='add') drop_res = [ Dropout(drop_res[j], name='drop_res-%d' % j) for j in range(n_layers) ] conv = [[ Conv1D(filter[j], window, activation='relu', name='conv%d-%d' % (i, j)) for j in range(n_layers) ] for i in range(n_clause)] drop_conv = [ Dropout(drop_conv[j], name='drop_conv-%d' % j) for j in range(n_layers) ] pool = MaxPooling1D(maxlen, padding='same', name='pool') drop_emb = Dropout(drop_emb, name='drop_emb') flatten = Flatten(name='flatten') dense_layers = [ Dense(n, activation='relu', name='dense%d' % i) for i, n in enumerate(dense_layers) ] drop_dense = [ Dropout(dp, name='drop_dense-%d' % j) for j, dp in enumerate(drop_dense) ] dense_out = Dense(n_relation, activation='softmax', name='dense_out') concat = Concatenate(name='concat') # Forward h = [embed(clause) for clause in inputs] if self.pe: h = [pe(hi) for hi in h] h = [drop_emb(hi) for hi in h] if sa_scale: h = [attn(hi) for hi in h] h = [drop_sa(hi) for hi in h] for j in range(n_layers): if residue: r = [reduce[i][j](h[i]) for i in range(n_clause)] r = [drop_res[j](ri) for ri in r] h = [conv[i][j](h[i]) for i in range(n_clause)] h = [drop_conv[j](hi) for hi in h] if residue: h = [add([h[i], r[i]]) for i in range(n_clause)] h = [pool(hi) for hi in h] h = [flatten(hi) for hi in h] h = concat(h + feature) for dense, drop in zip(dense_layers, drop_dense): h = drop(dense(h)) output = dense_out(h) super().__init__(inputs=inputs + feature, outputs=output) self.init_weights = self.get_weights()
def _build_layers_v2(self, input_dict, num_outputs, options): TRANSFORMER_SIMPLICIAL_DIM = options["custom_options"][ "transformer_simplicial_model_dim"] TRANSFORMER_MODEL_DIM = options["custom_options"][ "transformer_model_dim"] TRANSFORMER_NUM_HEADS = options["custom_options"][ "transformer_num_heads"] TRANSFORMER_DEPTH = options["custom_options"]["transformer_depth"] CONV_PADDING = options["custom_options"]["conv_padding"] NUM_VIRTUAL_ENTITIES = options["custom_options"][ "num_virtual_entities"] # For detailed comments see the base agent inputs = input_dict["obs"] sess = tf.get_default_session() K.set_session(sess) attention_layer = MultiHeadSelfAttentionZambaldi( name='self_attention', num_heads=TRANSFORMER_NUM_HEADS, use_masking=False, dropout=0, compression_window_size=None, num_virtual_entities=NUM_VIRTUAL_ENTITIES) attention_layer_2simplex = MultiHeadSelfAttentionSimplicial( name='self_2attention', num_heads=1, d_simp_model=TRANSFORMER_SIMPLICIAL_DIM, use_masking=False, dropout=0, compression_window_size=None, num_virtual_entities=NUM_VIRTUAL_ENTITIES) dense_layer1 = layers.Dense(TRANSFORMER_MODEL_DIM, activation='relu') dense_layer2 = layers.Dense(TRANSFORMER_MODEL_DIM) def transformer_block(input): a = LayerNormalization()(input) a1 = attention_layer( a ) # a1 = attention(h) has shape -1, seq_len, TRANSFORMER_MODEL_DIM a2 = attention_layer_2simplex( a) # shape -1, seq_len, TRANSFORMER_SIMPLICIAL_DIM a2 = LayerNormalization()(a2) ac = Concatenate()( [a1, a2] ) # shape -1, seq_len, TRANSFORMER_MODEL_DIM + TRANSFORMER_SIMPLICIAL_DIM b = dense_layer1(ac) b2 = dense_layer2(b) # b = ff(ac) r = layers.Add()([input, b2]) Hprime = LayerNormalization()(r) return Hprime # CONVOLUTIONS ------ # x = layers.Lambda(lambda x: x / 255)(inputs) # rescale RGB to [0,1] x = layers.Conv2D(12, (2, 2), activation='relu', padding=CONV_PADDING)(x) x = layers.Conv2D(24, (2, 2), activation='relu', padding=CONV_PADDING)( x) # output shape -1, num_rows, num_cols, 62 x = layers.Dense( TRANSFORMER_MODEL_DIM - 2, activation=None, use_bias=False )(x) # output shape -1, num_rows, num_cols, TRANSFORMER_MODEL_DIM-2 # POSITION EMBEDDING ----- # num_rows, num_cols, d_model = x.get_shape().as_list()[-3:] ps = np.zeros([num_rows, num_cols, 2], dtype=K.floatx()) # shape (12,13,2) for ty in range(num_rows): for tx in range(num_cols): ps[ty, tx, :] = [(2 / (num_rows - 1)) * ty - 1, (2 / (num_cols - 1)) * tx - 1] ps_expand = K.expand_dims(K.constant(ps), axis=0) # shape (1,num_rows,num_cols,2) ps_tiled = K.tile( ps_expand, [K.shape(x)[0], 1, 1, 1]) # shape (None,num_rows,num_cols,2) # (None,num_rows,num_cols,62) concatenate with (None,num_rows,num_cols,2) # to get (None,num_rows,num_cols,TRANSFORMER_MODEL_DIM) x = Concatenate(axis=3)([x, ps_tiled]) x = layers.Reshape( (num_rows * num_cols, d_model + 2))(x) # shape (None, num_rows*num_cols,d_model+2) # NOTE: the batch dimension is preserved by reshape, see https://www.tensorflow.org/api_docs/python/tf/keras/layers/Reshape # We now add some virtual entities, which are initialised randomly tokens = np.arange(NUM_VIRTUAL_ENTITIES).reshape( (1, NUM_VIRTUAL_ENTITIES)) # [[0,1,2,...,NUM_VIRTUAL_ENTITIES]] tokens = K.constant(tokens) ve = layers.Embedding( input_dim=NUM_VIRTUAL_ENTITIES, output_dim=d_model + 2)( tokens) # shape (1,NUM_VIRTUAL_ENTITIES,d_model+2) ve_tiled = K.tile(ve, [K.shape(x)[0], 1, 1]) x = Concatenate(axis=1)([x, ve_tiled]) # TRANSFORMER ----- for i in range(TRANSFORMER_DEPTH): x = transformer_block(x) # The output of the simplicial Transformer includes the virtual entities, # which we now want to remove. The current tensor is of shape # (None,num_rows*num_cols+NUM_VIRTUAL_ENTITIES,TRANSFORMER_MODEL_DIM) x = x[:, :-NUM_VIRTUAL_ENTITIES, :] # MAX-POOLING ----- # from p.4 "The E~ matrix, with shape Nxf is reudced to an f-dimensional vector by max-pooling # over the entity dimension. This pooled vector is then passed to a small MLP..." num_entities, d_model = x.get_shape().as_list()[-2:] x = layers.MaxPooling1D(pool_size=num_entities)(x) x = layers.Flatten()(x) # FULLY-CONNECTED LAYERS ---- x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) output_tensor = layers.Dense(4)(x) # final output is logits return output_tensor, x
content_weight = 0.025 #Weight of image content total_variation_weight = 10 ** -4 #Weight of image variation out_h = 400 #Output image height in pixels iterations = 20 #Number of transfer interations ''' Prepare images, model, and loss function. ''' #Calculate width to scale output image to a height of out_h. w, h = load_img(target_file).size out_w = int(w * out_h / h) #Load pre-processed images and prepare placeholder. target = K.constant(sdu.preprocess_image(target_file, out_h, out_w)) style = K.constant(sdu.preprocess_image(style_file, out_h, out_w)) combined = K.placeholder((1, out_h, out_w, 3)) #Combine images into a single tensor. input_tensor = K.concatenate([target, style, combined], axis = 0) #Load VGG19 network with ImageNet weights. model = vgg19.VGG19(input_tensor = input_tensor, weights = "imagenet", include_top = False) #Define general loss function. weights = [content_weight, style_weight, total_variation_weight] loss = sdu.general_loss(model, weights, combined, out_h, out_w) #Get gradients of the generated image.
F_u = Dropout(0.1)(F_u) F_u = WeightedAvgOverTime()(F_u) U = Add()([U, F_u]) #F_b = Reshape((feedback_b.shape[1],))(Embedding(book_num, feedback_b.shape[1], trainable=False, weights=[feedback_b])(b_input)) #F_b = Embedding(user_num+1, args.emb_dim, embeddings_initializer=Zeros(), embeddings_regularizer=l2(0.00001), mask_zero=True)(F_b) #F_b = Dropout(0.1)(F_b) #F_b = WeightedAvgOverTime()(F_b) #B = Add()([B, F_b]) pred = Dot(axes=-1)([U, B]) pred = Add()([pred, U_bias, B_bias]) pred = Lambda(lambda x: x + K.constant(rate_mean, dtype=K.floatx()))(pred) else: pred = Dot(axes=-1)([U, B]) pred = Add()([pred, U_bias, B_bias]) else: pred = Concatenate()([U, B]) for units in args.dense_units: pred = Dense(units, activation='selu')(pred) pred = Dropout(0.1)(pred) if args.norm_ans: pred = Dense(1, activation='selu')(pred) else: pred = Dense(1)(pred) pred = Add()([pred, U_bias, B_bias])
def DeepConv1DOptLearnerStaticArchitecture(param_trainable, init_wrapper, smpl_params, input_info, faces, emb_size=1000, input_type="3D_POINTS"): """ Optimised learner network architecture """ # An embedding layer is required to optimise the parameters optlearner_input = Input(shape=(1,), name="embedding_index") # Initialise the embedding layers emb_layers = init_emb_layers(optlearner_input, emb_size, param_trainable, init_wrapper) optlearner_params = Concatenate(name="parameter_embedding")(emb_layers) optlearner_params = Reshape(target_shape=(85,), name="learned_params")(optlearner_params) print("optlearner parameters shape: " +str(optlearner_params.shape)) #exit(1) # Ground truth parameters and point cloud are inputs to the model as well gt_params = Input(shape=(85,), name="gt_params") gt_pc = Input(shape=(6890, 3), name="gt_pc") print("gt parameters shape: " + str(gt_params.shape)) print("gt point cloud shape: " + str(gt_pc.shape)) # Compute the true offset (i.e. difference) between the ground truth and learned parameters pi = K.constant(np.pi) delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d")([gt_params, optlearner_params]) #delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d_no_mod")([gt_params, optlearner_params]) #delta_d = Lambda(lambda x: K.tf.math.floormod(x - pi, 2*pi) - pi, name="delta_d")(delta_d) # custom modulo 2pi of delta_d #delta_d = custom_mod(delta_d, pi, name="delta_d") # custom modulo 2pi of delta_d print("delta_d shape: " + str(delta_d.shape)) #exit(1) # Calculate the (batched) MSE between the learned parameters and the ground truth parameters false_loss_delta_d = Lambda(lambda x: K.mean(K.square(x), axis=1))(delta_d) print("delta_d loss shape: " + str(false_loss_delta_d.shape)) #exit(1) false_loss_delta_d = Reshape(target_shape=(1,), name="delta_d_mse")(false_loss_delta_d) print("delta_d loss shape: " + str(false_loss_delta_d.shape)) # Load SMPL model and get necessary parameters optlearner_pc = get_pc(optlearner_params, smpl_params, input_info, faces) # UNCOMMENT print("optlearner_pc shape: " + str(optlearner_pc.shape)) #exit(1) #optlearner_pc = Dense(6890*3)(delta_d) #optlearner_pc = Reshape((6890, 3))(optlearner_pc) # Get the (batched) Euclidean loss between the learned and ground truth point clouds pc_euclidean_diff = Lambda(lambda x: x[0] - x[1])([gt_pc, optlearner_pc]) pc_euclidean_dist = Lambda(lambda x: K.sum(K.square(x),axis=-1))(pc_euclidean_diff) print('pc euclidean dist '+str(pc_euclidean_dist.shape)) #exit(1) false_loss_pc = Lambda(lambda x: K.mean(x, axis=1))(pc_euclidean_dist) false_loss_pc = Reshape(target_shape=(1,), name="pc_mean_euc_dist")(false_loss_pc) print("point cloud loss shape: " + str(false_loss_pc.shape)) #exit(1) # Gather sets of points and compute their cross product to get mesh normals # In order of: right hand, right wrist, right forearm, right bicep end, right bicep, right shoulder, top of cranium, left shoulder, left bicep, left bicep end, left forearm, left wrist, left hand, # chest, belly/belly button, back of neck, upper back, central back, lower back/tailbone, # left foot, left over-ankle, left shin, left over-knee, left quadricep, left hip, right, hip, right, quadricep, right over-knee, right shin, right, over-ankle, right foot vertex_list = [5674, 5705, 5039, 5151, 4977, 4198, 411, 606, 1506, 1682, 1571, 2244, 2212, 3074, 3500, 460, 2878, 3014, 3021, 3365, 4606, 4588, 4671, 6877, 1799, 5262, 3479, 1187, 1102, 1120, 6740] #face_array = np.array([11396, 8620, 7866, 5431, 6460, 1732, 4507]) pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network vertex_diff_NOGRAD = Lambda(lambda x: K.tf.gather(x, np.array(vertex_list).astype(np.int32), axis=-2))(pc_euclidean_diff_NOGRAD) print("vertex_diff_NOGRAD shape: " + str(vertex_diff_NOGRAD.shape)) vertex_diff_NOGRAD = Flatten()(vertex_diff_NOGRAD) #exit(1) face_array = np.array([[face for face in faces if vertex in face][0] for vertex in vertex_list]) # only take a single face for each vertex print("face_array shape: " + str(face_array.shape)) gt_normals = get_mesh_normals(gt_pc, face_array, layer_name="gt_cross_product") print("gt_normals shape: " + str(gt_normals.shape)) opt_normals = get_mesh_normals(optlearner_pc, face_array, layer_name="opt_cross_product") print("opt_normals shape: " + str(opt_normals.shape)) #exit(1) # Learn the offset in parameters from the difference between the ground truth and learned mesh normals diff_normals = Lambda(lambda x: K.tf.cross(x[0], x[1]), name="diff_cross_product")([gt_normals, opt_normals]) diff_normals_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_normals) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network diff_angles = Lambda(lambda x: K.tf.subtract(x[0], x[1]), name="diff_angle")([gt_normals, opt_normals]) diff_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_angles) diff_angles_norm_NOGRAD = Lambda(lambda x: K.tf.norm(x, axis=-1), name="diff_angle_norm")(diff_angles_NOGRAD) dist_angles = Lambda(lambda x: K.mean(K.square(x), axis=-1), name="diff_angle_mse")(diff_angles) dist_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(dist_angles) print("diff_angles shape: " + str(diff_angles.shape)) print("dist_angles shape: " + str(dist_angles.shape)) #pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network #print("diff_normals_NOGRAD shape: " + str(diff_normals_NOGRAD.shape)) diff_normals_NOGRAD = Flatten()(diff_normals_NOGRAD) diff_angles_NOGRAD = Flatten()(diff_angles_NOGRAD) mesh_diff_NOGRAD = Concatenate()([diff_normals_NOGRAD, dist_angles_NOGRAD]) if input_type == "3D_POINTS": #optlearner_architecture = Dense(2**9, activation="relu")(vertex_diff_NOGRAD) optlearner_architecture = Dense(2**7, activation="relu")(vertex_diff_NOGRAD) if input_type == "MESH_NORMALS": #optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_norm_NOGRAD) #optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_NOGRAD) #optlearner_architecture = Dense(2**9, activation="relu")(mesh_diff_NOGRAD) optlearner_architecture = Dense(2**7, activation="relu")(mesh_diff_NOGRAD) #optlearner_architecture = BatchNormalization()(optlearner_architecture) #optlearner_architecture = Dropout(0.5)(optlearner_architecture) print('optlearner_architecture shape: '+str(optlearner_architecture.shape)) optlearner_architecture = Reshape((optlearner_architecture.shape[1].value, 1))(optlearner_architecture) print('optlearner_architecture shape: '+str(optlearner_architecture.shape)) optlearner_architecture = Conv1D(64, 5, activation="relu")(optlearner_architecture) optlearner_architecture = BatchNormalization()(optlearner_architecture) optlearner_architecture = MaxPooling1D(3)(optlearner_architecture) optlearner_architecture = Conv1D(128, 5, activation="relu")(optlearner_architecture) optlearner_architecture = BatchNormalization()(optlearner_architecture) optlearner_architecture = MaxPooling1D(2)(optlearner_architecture) optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture) optlearner_architecture = BatchNormalization()(optlearner_architecture) optlearner_architecture = MaxPooling1D(2)(optlearner_architecture) optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture) optlearner_architecture = BatchNormalization()(optlearner_architecture) #optlearner_architecture = MaxPooling1D(2)(optlearner_architecture) optlearner_architecture = AveragePooling1D(2)(optlearner_architecture) print('optlearner_architecture shape: '+str(optlearner_architecture.shape)) optlearner_architecture = Flatten()(optlearner_architecture) print('optlearner_architecture shape: '+str(optlearner_architecture.shape)) #optlearner_architecture = Dropout(0.5)(optlearner_architecture) optlearner_architecture = Dense(2**7, activation="relu")(optlearner_architecture) print('optlearner_architecture shape: '+str(optlearner_architecture.shape)) #delta_d_hat = Dense(85, activation=pos_scaled_tanh, name="delta_d_hat")(optlearner_architecture) delta_d_hat = Dense(85, activation="linear", name="delta_d_hat")(optlearner_architecture) #delta_d_hat = Dense(85, activation=centred_linear, name="delta_d_hat")(optlearner_architecture) print('delta_d_hat shape: '+str(delta_d_hat.shape)) #exit(1) # Calculate the (batched) MSE between the learned and ground truth offset in the parameters delta_d_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(delta_d) false_loss_delta_d_hat = Lambda(lambda x: K.mean(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat]) #false_loss_delta_d_hat = Lambda(lambda x: K.sum(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat]) #false_loss_delta_d_hat = Lambda(lambda x: mape(x[0], x[1]))([delta_d_NOGRAD, delta_d_hat]) false_loss_delta_d_hat = Reshape(target_shape=(1,), name="delta_d_hat_mse")(false_loss_delta_d_hat) print("delta_d_hat loss shape: " + str(false_loss_delta_d_hat.shape)) #false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat) false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat, average=False) false_sin_loss_delta_d_hat = Lambda(lambda x: x, name="delta_d_hat_sin_output")(false_sin_loss_delta_d_hat) print("delta_d_hat sin loss shape: " + str(false_sin_loss_delta_d_hat.shape)) # Prevent model from using the delta_d_hat gradient in final loss delta_d_hat_NOGRAD = Lambda(lambda x: K.stop_gradient(x), name='optlearner_output_NOGRAD')(delta_d_hat) # False loss designed to pass the learned offset as a gradient to the embedding layer false_loss_smpl = Multiply(name="smpl_diff")([optlearner_params, delta_d_hat_NOGRAD]) print("smpl loss shape: " + str(false_loss_smpl.shape)) #return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, delta_d_hat_NOGRAD] return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, dist_angles]
def build(self, input_shape): param_shape_unshared = list(input_shape[1:]) param_shape_shared = list(input_shape[1:]) param_broadcast_unshared = [False] * len(param_shape_unshared) param_broadcast_shared = [False] * len(param_shape_shared) if self.shared_axes is not None: for i in self.shared_axes: param_shape_shared[i - 1] = 1 param_broadcast_shared[i - 1] = True self.a_param_broadcast = param_broadcast_shared self.k_param_broadcast = param_broadcast_shared self.n_param_broadcast = param_broadcast_shared self.z_param_broadcast = param_broadcast_shared a_param_shape = param_shape_shared k_param_shape = param_shape_shared n_param_shape = param_shape_shared z_param_shape = param_shape_shared if not self.a_shared: a_param_shape = param_shape_unshared self.a_param_broadcast = param_broadcast_unshared if not self.k_shared: k_param_shape = param_shape_unshared self.k_param_broadcast = param_broadcast_unshared if not self.n_shared: n_param_shape = param_shape_unshared self.n_param_broadcast = param_broadcast_unshared if not self.z_shared: z_param_shape = param_shape_unshared self.z_param_broadcast = param_broadcast_unshared self.a = self.add_weight(shape=a_param_shape, name='a', initializer=self.a_initializer, regularizer=self.a_regularizer, constraint=self.a_constraint) self.k = self.add_weight(shape=k_param_shape, name='k', initializer=self.k_initializer, regularizer=self.k_regularizer, constraint=self.k_constraint) self.n = self.add_weight(shape=n_param_shape, name='n', initializer=self.n_initializer, regularizer=self.n_regularizer, constraint=self.n_constraint) if not self.z_one: self.z = self.add_weight(shape=z_param_shape, name='z', initializer=self.z_initializer, regularizer=self.z_regularizer, constraint=self.z_constraint) else: self.z = K.constant(1.) # Set input spec axes = {} if self.shared_axes: for i in range(1, len(input_shape)): if i not in self.shared_axes: axes[i] = input_shape[i] self.input_spec = InputSpec(ndim=len(input_shape), axes=axes) self.built = True
dataset[np.arange(n), np.random.choice(range(len(probs)), p=probs, size=n, replace=True)] = 1 # model = Sequential() # model.add(Dense(units=len(probs), activation='softmax', input_dim=1, activity_regularizer=IndepedentRegularizer((2,2,2), weight=1.))) # model.compile(optimizer='sgd', loss='categorical_crossentropy') # model.fit(np.zeros((n, 1)), dataset, batch_size=10, verbose=False) # print(entropy(probs, model.predict(np.array([[0]]))[0])) # model = Sequential() # model.add(Dense(units=len(probs), activation='softmax', input_dim=1)) # model.compile(optimizer='sgd', loss='categorical_crossentropy') # model.fit(np.zeros((n, 1)), dataset, batch_size=10, verbose=False) # print(entropy(probs, model.predict(np.array([[0]]))[0])) reg = IndepedentRegularizer((2, 2, 2, 2, 2, 2), 10.) freqs = K.constant((1 + np.sum(dataset, axis=0)) / (n + len(probs))) prob_tensor = K.variable(np.sum(dataset, axis=0) / n) prob_tensor = prob_tensor / tf.reduce_sum(prob_tensor) optimizer = tf.train.GradientDescentOptimizer(0.01) train = optimizer.minimize( reg(prob_tensor) + kullback_leibler_divergence(freqs, prob_tensor)) init = tf.initialize_all_variables() with tf.Session() as session: session.run(init) for step in range(1000): session.run(train) print("step", step, "entropy:", entropy(probs, session.run(prob_tensor))) print(session.run(prob_tensor))
self.loss_value = None self.grad_values = None return grad_values # This is the path to the image you want to transform. TARGET_IMG = 'tiger.jpeg' # This is the path to the style image. REFERENCE_STYLE_IMG = 'style3.png' width, height = load_img(TARGET_IMG).size img_height = 320 img_width = int(width * img_height / height) target_image = K.constant(preprocess_image(TARGET_IMG, height=img_height, width=img_width)) style_image = K.constant(preprocess_image(REFERENCE_STYLE_IMG, height=img_height, width=img_width)) # Placeholder for our generated image generated_image = K.placeholder((1, img_height, img_width, 3)) # Combine the 3 images into a single batch input_tensor = K.concatenate([target_image, style_image, generated_image], axis=0) model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) layers = dict([(layer.name, layer.output) for layer in model.layers])
def VSR(scale=4, depth=7, num_blocks=8, nD=4, name=None): if depth % 2 == 0: raise ValueError("bad depth") Input_Tensor = Input((depth, None, None, 3)) X_in = Normalization()(Input_Tensor) X = ZeroPadding3D(padding=(0, 3, 3))(X_in) X = Conv3D(filters=64, kernel_size=(1, 7, 7), strides=(1, 1, 1), padding='valid')(X) # residual dense blocks for i in range(num_blocks): X = ResidualDenseBlock(X, 64, 64, 16, nD) pass # depth turn to 1 num_conv = int((depth - 1) / 2) nF = 128 nG = 64 for i in range(num_conv): T = BatchNormalization(axis=4)(X) T = LeakyReLU(alpha=0.1)(T) T = Conv3D(filters=nF, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid')(T) T = BatchNormalization(axis=4)(T) T = LeakyReLU(alpha=0.1)(T) T = ZeroPadding3D(padding=(0, 1, 1))(T) T = Conv3D(filters=nG, kernel_size=(3, 3, 3), strides=(1, 1, 1), padding='valid')(T) S = Lambda(lambda x: x[:, 1:-1])(X) X = concatenate([S, T], axis=4) nF += nG pass X = BatchNormalization(axis=4)(X) X = LeakyReLU(alpha=0.1)(X) X = ZeroPadding3D(padding=(0, 1, 1))(X) X = Conv3D(filters=256, kernel_size=(1, 3, 3), strides=(1, 1, 1), padding='valid')(X) X = LeakyReLU(alpha=0.1)(X) R = Conv3D(filters=256, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid')(X) R = LeakyReLU(alpha=0.1)(R) R = Conv3D(filters=3 * scale * scale, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid')(R) F = Conv3D(filters=512, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid')(X) F = LeakyReLU(alpha=0.1)(F) F = Conv3D(filters=1 * 5 * 5 * scale * scale, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid')(F) shape = (K.shape(F)[0], K.shape(F)[1], K.shape(F)[2], K.shape(F)[3], 25, scale * scale) F = Lambda(K.reshape, arguments={'shape': shape})(F) F = Softmax(axis=4)(F) # dynamic filter filter_localexpand_np = np.reshape(np.eye(25, 25), (5, 5, 1, 25)) filter_localexpand = K.constant(filter_localexpand_np, dtype='float32') x_channals = [] for c in range(3): x = Lambda(lambda x: x[:, depth // 2:depth // 2 + 1, :, :, c])(X_in) f = Lambda(lambda x: x[:, 0, :, :, :, :])(F) x = Lambda(tf.transpose, arguments={'perm': [0, 2, 3, 1]})(x) x_localexpand = Lambda(K.conv2d, arguments={'kernel':filter_localexpand, \ 'strides':(1, 1), 'padding':'same'})(x) x_localexpand = Lambda(tf.expand_dims, arguments={'axis': 3})(x_localexpand) x = Lambda(tf.matmul, arguments={'b': f})(x_localexpand) x = Lambda(tf.squeeze, arguments={'axis': 3})(x) x = SubpixelConv2D(scale=scale)(x) x_channals.append(x) pass x = concatenate(x_channals, axis=3) x = Lambda(tf.expand_dims, arguments={'axis': 1})(x) # depth to space 3D r_shape = K.shape(R) shape = (r_shape[0] * r_shape[1], r_shape[2], r_shape[3], r_shape[4]) r = Lambda(K.reshape, arguments={'shape': shape})(R) y = Lambda(tf.depth_to_space, arguments={'block_size': scale})(r) y_shape = K.shape(y) shape = (r_shape[0], r_shape[1], y_shape[1], y_shape[2], y_shape[3]) r = Lambda(K.reshape, arguments={'shape': shape})(y) # add G = layers.add([x, r]) G = Lambda(lambda x: x[:, 0, :, :, :])(G) Output_Tensor = Denormalization()(G) if name == None: name = "VSR-" + str(num_blocks) + "-" + str(nD) + "-x" + str(scale) pass model = Model(Input_Tensor, Output_Tensor, name=name) return model
def loss_func_ck_default(): return K.constant(0.0)
def __init__(self, select, **kwargs): super(Selector, self).__init__(**kwargs) self.select = select self.select_neuron = K.constant(value=self.select)
def one_elem_loss_generator(id_elem, elem_param, basis_info, model_info): """ build loss function for one element :param id_elem: id of current element and sub-model :param elem_param: tuple, (n_modes, n_quads, lambda_coeff, bc, jacobian, domain, ck_k) where n_modes is the number of modes in each element n_quads is the number of quadrature points in each element lambda_coeff is the lambda coefficient bc : numpy array, shape: (2,2); bc[:, 0] contains coordinates of 2 boundaries bc[:, 1] contains Dirivhlet BC values of 2 boundaries jacobian: vector containing jacobians of each element, shape: (n_elem,) domain: vector contains element boundary coordinates, shape: (n_elem+1,) ck_k : k value in C_k continuity :param basis_info: tuple, (B, Bd, W, B_bound) where B : basis matrix, shape: (n_quads, n_modes) Bd : basis derivative matrix, shape: (n_quads, n_modes) W : weight matrix, shape: (1, n_quads) B_bound : matrix of basis values on x=-1, 1; shape: (2, n_modes) :param model_info: tuple, (model, sub_model_set) where model: multi_element_model for n_elem elements, overall model of n_elem sub-models sub_model_set: list of sub-models for each element, shape: (n_elem,) :return: loss function for the sub-model of this element if this is the first element, will compute equation residual of this element, and also of element-boundary C^k continuity conditions, as well as global boundary condition loss if this is not the first element, will compute only the equation residual of this element """ n_modes, n_quads, lambda_coeff, bc, jacob, this_domain, ck_k = elem_param n_elem = jacob.shape[0] # number of elements global_model, sub_model_list = model_info # global_model contains the overall model this_sub_model = sub_model_list[id_elem] # sub_model for this element first_sub_model = sub_model_list[0] # sub_model for first element last_sub_model = sub_model_list[n_elem - 1] # sub_model for last element B_trans, Bd_trans, W_mat_A, B_bound = basis_info # B_trans contains basis function matrix, shape: (n_quads, n_modes) # Bd_trans contains basis derivative matrix, shape: (n_quads, n_modes) # W_mat_A contains weight matrix, shape: (1, n_quads) # B_bound contains boundary basis matrix, shape: (2, n_modes) W_mat = np.zeros((n_elem, n_quads)) W_A = np.zeros((n_elem, n_quads)) for i in range(n_elem): W_mat[i, :] = W_mat_A[0, :] * jacob[i] W_A[i, :] = W_mat_A[0, :] # now W_mat contains jacobian * W matrix, shape: (n_elem, n_quads) # W_A contains W matrix, shape: (n_elem, n_quads) B_bound_trans = np.copy(B_bound) # shape: (2, n_modes) B_bound_trans[0, :] = -B_bound_trans[0, :] # negate value at x=-1, because in weak form we have -phi(-1)*du/dx|_{x=-1} B_bound_left = np.zeros((n_elem, n_modes, 1)) # shape: (n_elem,n_quads,1) B_bound_right = np.zeros((n_elem, n_modes, 1)) # shape: (n_elem,n_quads,1) B_bound_left[0, :, 0] = B_bound_trans[ 0, :] # modes of first element, left boundary x=a B_bound_right[-1, :, 0] = B_bound_trans[ 1, :] # modes of right element, right boundary x=b ## ============================================## # generate Keras tensors from these matrices or vectors B_tensor = K.constant( B_trans) # in standard element, shape: (n_quads, n_modes) W_mat_tensor = K.constant( W_mat) # shape of W_mat_tensor: (n_elem, n_quads) W_mat_A_tensor = K.constant(W_A) # shape : (n_elem, n_quads) Bd_tensor = K.constant( Bd_trans) # in standard element, shape; (n_quads, n_modes) # B_bound_tensor = K.constant(B_bound_trans) # shape: (2, n_modes) B_bound_left_tensor = K.constant( B_bound_left) # shape: (n_elem, n_modes, 1) B_bound_right_tensor = K.constant( B_bound_right) # shape: (n_elem, n_modes, 1) # ======================================= # # element boundary du/dx # Zb = np.zeros((2, 1)) # Zb[0, 0] = this_domain[id_elem] # Zb[1, 0] = this_domain[id_elem+1] # now Zb_elem contains coordinates of end points for this element # Zb_tensor = K.constant(Zb) # Zb_input_tensor = keras.Input(tensor = Zb_tensor) # now Zb_input_tensor contains input tensor for this sub-model # ======================================= # # boundary conditions for domain ZL = np.zeros((1, 1)) ZL[0, 0] = bc[0, 0] # left boundary coordinate ZL_tensor = K.constant(ZL) # convert ZL_tensor into Keras input tensor ZL_input_tensor = keras.Input(tensor=ZL_tensor) ZR = np.zeros((1, 1)) ZR[0, 0] = bc[1, 0] # right boundary coordinate ZR_tensor = K.constant(ZR) ZR_input_tensor = keras.Input(tensor=ZR_tensor) BCL = np.zeros((1, 1)) BCL[0, 0] = bc[0, 1] # left boundary condition BCL_tensor = K.constant(BCL) # shape: (1,1) BCR = np.zeros((1, 1)) BCR[0, 0] = bc[1, 1] # right boundary condition BCR_tensor = K.constant(BCR) # shape: (1,1) # ====================================== # # continuity across element boundaries Zb_elem = np.zeros((n_elem, 2, 1)) for i in range(n_elem): Zb_elem[i, 0, 0] = this_domain[i] Zb_elem[i, 1, 0] = this_domain[i + 1] Zb_elem_tensor = [] for i in range(n_elem): temp_tensor = K.constant(Zb_elem[i]) # shape (2,1) temp_input_tensor = keras.Input(tensor=temp_tensor) Zb_elem_tensor.append(temp_input_tensor) # now Zb_elem_tensor contains list of element-boundary coordinate tensors # element-boundary value difference bound_assembly = np.zeros((n_elem - 1, n_elem * 2)) for i in range(n_elem - 1): bound_assembly[ i, 2 * i + 1] = 1.0 # right boundary of i-th element, the one on the left bound_assembly[i, 2 * ( i + 1 )] = -1.0 # left boundary of (i+1)-th element, the one on the right bound_assembly_tensor = K.constant( bound_assembly) # shape: (n_elem-1, n_elem*2) # to be used for computing different between values across element boundaries # ============================= # # element boundary mode assembly matrix and tensor bmode_assembler = np.zeros((n_elem + 1, 2 * n_elem)) bmode_assembler[0, 0] = 1.0 bmode_assembler[-1, -1] = 1.0 for i in range(n_elem - 1): bmode_assembler[i + 1, 2 * i + 1] = 1.0 # i-th element, right boundary bmode_assembler[i + 1, 2 * (i + 1)] = 1.0 # (i+1)-th element, left boundary # now bmode_assembler contains assembler matrix bmode_assembler_tensor = K.constant( bmode_assembler) # Keras tensor, shape: (n_elem+1, 2*n_elem) # ============================= # def loss_generator_Ck(ck): """ Ck loss function generator :param ck: integer, >= 0, k value in C_k continuity :return: loss function for C_k """ def loss_func_c0(): """ compute loss across element boundary for C^0 continuity across element boundaries :return: loss value associated with C0 continuity condition """ # now Zb_elem_tensor contains a list of (2,1) element boundary coordinate tensors output_tensor = global_model(Zb_elem_tensor) # now output_tensor contains list of output tensors, shape: (n_elem, 2, 1) temp_out_tensor = K.concatenate(output_tensor, 0) # concatenate list tensors # now temp_out_tensor has shape (n_elem*2, 1) C0_residual_tensor = K.dot(bound_assembly_tensor, temp_out_tensor) # C0_residual_tensor has shape: (n_elem-1, 1) this_loss = K.sum(K.square(C0_residual_tensor)) return this_loss def loss_func_c1(): """ compute loss across element boundary for C^1 continuity across element boundaries :return: loss value """ output_tensor = global_model(Zb_elem_tensor) # now output_tensor contains list of output tensors, shape: (n_elem, 2, 1) grad_dudx = K.gradients(output_tensor, Zb_elem_tensor) # now grad_dudx contains gradients, # shape is (n_elem, 2, 1) # grad_dudx is a list of (2,1) arrays C0_concat = K.concatenate(output_tensor, 0) # shape (n_elem*2, 1) C1_concat = K.concatenate(grad_dudx, 0) # shape (n_elem*2, 1) C0_residual = K.dot(bound_assembly_tensor, C0_concat) # shape: (n_elem-1, 1) C1_residual = K.dot(bound_assembly_tensor, C1_concat) # shape: (n_elem-1, 1) this_loss = K.sum(K.square(C0_residual)) + K.sum( K.square(C1_residual)) return this_loss def loss_func_c2(): """ loss for C^2 continuity across element boundaries :return: loss value """ ub = global_model( Zb_elem_tensor ) # ub constains element boundary value tensor, shape: (n_elem, 2, 1) dub_dx = K.gradients( ub, Zb_elem_tensor ) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1) dub_dx_2 = K.gradients(dub_dx, Zb_elem_tensor) # dud_dx_2 contains d^2u/dx^2 tensor on boundary, shape: (n_elem,2,1) # Note: tensorflow.gradients sums up all w.r.t. to list of output tensors # but since only du1/dx1, du2/dx2, du3/dx3 ... is non-zero, while all du_i/dx_j=0 if i!=j # dudx effectively contains only du_i/dx_i, for i=0, ..., n_elem-1 # similarly, for second derivatives, du_dx_2 contains only d^2 u_i/dx_i^2, for i=0,...,n_elem-1 only C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1) C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1) C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1) C0_residual = K.dot(bound_assembly_tensor, C0_concat) # shape: (n_elem-1, 1) C1_residual = K.dot(bound_assembly_tensor, C1_concat) # shape: (n_elem-1, 1) C2_residual = K.dot(bound_assembly_tensor, C2_concat) # shape: (n_elem-1, 1) this_loss = K.sum(K.square(C0_residual)) + K.sum( K.square(C1_residual)) + K.sum(K.square(C2_residual)) return this_loss def loss_func_c3(): """ compute loss associated with C^3 continuity :return: loss value """ ub = global_model( Zb_elem_tensor ) # ub constains element boundary value tensor, shape: (n_elem, 2, 1) dub_dx = K.gradients( ub, Zb_elem_tensor ) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1) dub_dx_2 = K.gradients( dub_dx, Zb_elem_tensor ) # second derivative tensor, shape: (n_elem,2,1) dub_dx_3 = K.gradients( dub_dx_2, Zb_elem_tensor) # third derivative tensor, shape: (n_elem,2,1) C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1) C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1) C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1) C3_concat = K.concatenate(dub_dx_3, 0) # shape: (n_elem*2,1) C0_residual = K.dot(bound_assembly_tensor, C0_concat) # shape: (n_elem-1, 1) C1_residual = K.dot(bound_assembly_tensor, C1_concat) # shape: (n_elem-1, 1) C2_residual = K.dot(bound_assembly_tensor, C2_concat) # shape: (n_elem-1, 1) C3_residual = K.dot(bound_assembly_tensor, C3_concat) # shape: (n_elem-1, 1) this_loss = K.sum(K.square(C0_residual)) + K.sum(K.square(C1_residual)) \ + K.sum(K.square(C2_residual)) + K.sum(K.square(C3_residual)) return this_loss def loss_func_c4(): """ compute loss associated with C^4 continuity :return: loss value """ ub = global_model( Zb_elem_tensor ) # ub constains element boundary value tensor, shape: (n_elem, 2, 1) dub_dx = K.gradients( ub, Zb_elem_tensor ) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1) dub_dx_2 = K.gradients( dub_dx, Zb_elem_tensor ) # second derivative tensor, shape: (n_elem,2,1) dub_dx_3 = K.gradients( dub_dx_2, Zb_elem_tensor) # third derivative tensor, shape: (n_elem,2,1) dub_dx_4 = K.gradients( dub_dx_3, Zb_elem_tensor ) # fourth derivative tensor, shape: (n_elem,2,1) C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1) C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1) C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1) C3_concat = K.concatenate(dub_dx_3, 0) # shape: (n_elem*2,1) C4_concat = K.concatenate(dub_dx_4, 0) # shape: (n_elem*2, 1) C0_residual = K.dot(bound_assembly_tensor, C0_concat) # shape: (n_elem-1, 1) C1_residual = K.dot(bound_assembly_tensor, C1_concat) # shape: (n_elem-1, 1) C2_residual = K.dot(bound_assembly_tensor, C2_concat) # shape: (n_elem-1, 1) C3_residual = K.dot(bound_assembly_tensor, C3_concat) # shape: (n_elem-1, 1) C4_residual = K.dot(bound_assembly_tensor, C4_concat) # shape: (n_elem-1, 1) this_loss = K.sum(K.square(C0_residual)) + K.sum(K.square(C1_residual)) \ + K.sum(K.square(C2_residual)) + K.sum(K.square(C3_residual)) \ + K.sum(K.square(C4_residual)) return this_loss def loss_func_ck_default(): return K.constant(0.0) # ++++++++++++ # if ck == 0: return loss_func_c0 elif ck == 1: return loss_func_c1 elif ck == 2: return loss_func_c2 elif ck == 3: return loss_func_c3 elif ck == 4: return loss_func_c4 else: print( "ERROR: loss_generator_ck() -- C^k continuity with (infinity > k > 4) is not implemented!\n" ) return loss_func_ck_default # +++++++++++++ # The_Loss_Func_Ck = loss_generator_Ck(ck_k) # now The_Loss_Func_Ck is the loss function for computing Ck continuity loss # across element boundaries # ======================================= # def Equation_Residual(left_tensor_pair, right_tensor_pair): """ actual computation of residual tensor for equation for all elements :param: left_tensor_pair: tuple, (BCL_in_tensor, BCL_out_tensor) where BCL_in_tensor is the input tensor for left domain boundary, shape: (1,1) BCL_out_tensor is the output tensor for left domain boundary, shape: (1,1) :param: right_tensor_pair: tuple, (BCR_in_tensor, BCR_out_tensor) where BCR_in_tensor is the input tensor for right domain boundary, shape: (1,1) BCR_out_tensor is the output tensor for right domain boundary, shape: (1,1) :return: equation residual tensor for all elements Note: when using tensorflow.gradients to compute derivative of a list of output tensors (u_i) with respect to a list of input tensors (x_j), the resultant gradient is a list with \sum_{i=0}^{N-1} du_i/dx_j, for j = 0, 1, ..., M-1. Note that it is summed over i. So the length of resultant list is equal to the length of the list of input tensors. In this code, since different sub-models are not connected, i.e. du_i/dx_j = 0 if i != j, the resultant sum will have no effect. So the resultant gradient is effectively du_i/dx_i, i=0, ..., M-1. """ # now global_model contains entire model # sub_model_list contains the list of sub-models # first_sub_model contains the DNN for the first element # last_sub_model contains the DNN for the last element # \sum du/dx * d_phi/dx*jacobian * w = \sum du/dx * d_phi/dxi * w # where x is physical coordinate, xi is coordinate in standard element In_tensors = global_model.inputs # list of input tensors, shape: (n_elem, n_quads, 1) Out_tensors = global_model.outputs # list of output tensors, shape: (n_elem, n_quads, 1) dudx = K.gradients( Out_tensors, In_tensors) # list of du/dx tensors, shape: (n_elem, n_quads, 1) dudx_con = K.concatenate(dudx, 0) # shape: (n_elem*n_quads, 1) dudx_1 = K.reshape(dudx_con, (n_elem, n_quads)) # shape: (n_elem, n_quads) dudx_w = dudx_1 * W_mat_A_tensor # element wise multiply, shape: (n_elem, n_quads) T1 = K.dot(dudx_w, Bd_tensor) # shape: (n_elem, n_modes) # Note that Bd_tensor contains dphi/dxi in standard element, shape: (n_quads,n_modes) # dphi/dx in physical space is dphi/dxi / jacobian # now T1[n_modes, n_elem] contains first term in weak form # lambda_coeff* \sum u*basis*jacobian * w # now Out_tensors contains u on all quadrature points of all elements, list of (n_quads,1) tensors # first concatenate them into a single tensor u_con = K.concatenate(Out_tensors, 0) # shape: (n_elem*n_quads, 1) u_1 = K.reshape(u_con, (n_elem, n_quads)) # shape: (n_elem, n_quads) temp_v1 = lambda_coeff * u_1 * W_mat_tensor # note: W_mat_tensor includes jacobian, shape: (n_elem,n_quads) T2 = K.dot(temp_v1, B_tensor) # shape: (n_elem, n_modes) # now T2 contains second term, shape: (n_elem, n_modes) # source term \sum f(x) * basis * jacobian * w # Note: list of label tensors is contained in global_model.targets target_tensors = global_model.targets # list of label tensors, shape: (n_elem, n_quads, ?) label_tensors = K.concatenate(target_tensors, 0) # shape: (n_elem*n_quads, ?) y_true_0 = label_tensors[:, 0] # shape: (n_elem*n_quads,) y_true_1 = K.reshape(y_true_0, (n_elem, n_quads)) # shape: (n_elem, n_quads) temp_v2 = y_true_1 * W_mat_tensor # temp_v2 has shape (n_elem, n_quads) T3 = K.dot(temp_v2, B_tensor) # shape: (n_elem, n_modes) # now T3 contains source term, shape (n_elem, n_modes) # boundary terms left_in_tensor, left_out_tensor = left_tensor_pair right_in_tensor, right_out_tensor = right_tensor_pair dudx_a = K.gradients( left_out_tensor, left_in_tensor)[0] # gradient on left boundary, shape: (1,1) dudx_b = K.gradients( right_out_tensor, right_in_tensor)[0] # gradient on right boundary, shape: (1,1) # Note: returned data from tensorflow.gradients is a list of tensors, even though the length # of the list may be 1. So we need the "...[0]" in the above to get the actual tensor T_bound_left_1 = K.dot(B_bound_left_tensor, dudx_a) # shape: (n_elem,n_modes,1) T_bound_left = K.squeeze(T_bound_left_1, -1) # shape: (n_elem, n_modes) T_bound_right_1 = K.dot(B_bound_right_tensor, dudx_b) # shape: (n_elem, n_modes, 1) T_bound_right = K.squeeze(T_bound_right_1, -1) # shape: (n_elem, n_modes) T_bound_elem = T_bound_left + T_bound_right # shape: (n_elem, n_modes) # now T_bound_elem contains contribution of boundary terms # -dudx(a)*phi(a) + dudx(b)*phi(b) # loss T_tot = T1 + T2 + T3 - T_bound_elem # now T_tot has shape: (n_elem, n_modes) return T_tot def The_Loss_Func(y_true, y_pred): """ actual computation of loss function for ordinary element, not the first element only compute the residual loss of equation for current element :param y_true: label data :param y_pred: preduction data :return: loss value """ # DNN corresponding to other elements will simply return 0 for loss # DNN corresponding to first element will compute the loss for all elements this_loss = K.constant(0.0) return this_loss def The_First_Loss_Func(y_true, y_pred): """ actual computation of loss for all elements compute residual loss of equation for all elements, and loss in continuity across element boundary, and loss in boundary condition of domain :param y_true: :param y_pred: :return: """ # ========================== # # domain boundary condition BCL_out = first_sub_model(ZL_input_tensor) # shape: (1,1) T_bc_L = BCL_out - BCL_tensor # shape: (1,1) BCR_out = last_sub_model(ZR_input_tensor) # shape: (1,1) T_bc_R = BCR_out - BCR_tensor # shape: (1,1) # ========================= # # equation residual bcl_pair = (ZL_input_tensor, BCL_out) bcr_pair = (ZR_input_tensor, BCR_out) T_tot = Equation_Residual(bcl_pair, bcr_pair) # now T_tot contains residual tensor for equation of all elements, shape: (n_elem, n_modes) # ========================== # # extract element boundary and interior modes # assemble boundary modes bmode = K.slice(T_tot, (0, 0), (n_elem, 2)) # bmode contains boundary mode contributions, shape: (n_elem, 2) interior_mode = K.slice(T_tot, (0, 2), (n_elem, n_modes - 2)) # interior_mode contains interior mode contributions, shape: (n_elem, n_modes-2) bmode_2 = K.reshape(bmode, (n_elem * 2, 1)) # shape (n_elem*2,1) bmode_glob = K.dot(bmode_assembler_tensor, bmode_2) # now bmode_glob contains global mode contributions, shape: (n_elem+1,1) # interior_mode contains interior mode contributions value_1 = K.flatten(bmode_glob) value_2 = K.flatten(interior_mode) value = K.concatenate((value_1, value_2)) # now value contains all global modes, shape: (n_elem+1+n_elem*(n_modes-2),), 1D array # ========================= # # C^k continuity across element boundary Ck_loss = The_Loss_Func_Ck() # now Ck_loss contains loss value for C_k continuity across element boundaries # ========================= # this_loss = K.mean(K.square(value)) \ + (K.sum(K.square(T_bc_L)) + K.sum(K.square(T_bc_R)) + Ck_loss) return this_loss # ====================================== # if id_elem == 0: return The_First_Loss_Func else: return The_Loss_Func
def attention(self, pre_q, pre_v, pre_k, out_seq_len: int, d_model: int, training=None, padding_mask=None): """ Calculates the output of the attention once the affine transformations of the inputs are done. Here's the shapes of the arguments: :param pre_q: (batch_size, q_seq_len, num_heads, d_model // num_heads) :param pre_v: (batch_size, v_seq_len, num_heads, d_model // num_heads) :param pre_k: (batch_size, k_seq_len, num_heads, d_model // num_heads) :param out_seq_len: the length of the output sequence :param d_model: dimensionality of the model (by the paper) :param training: Passed by Keras. Should not be defined manually. Optional scalar tensor indicating if we're in training or inference phase. """ # shaping Q and V into (batch_size, num_heads, seq_len, d_model//heads) q = K.permute_dimensions(pre_q, [0, 2, 1, 3]) v = K.permute_dimensions(pre_v, [0, 2, 1, 3]) if self.compression_window_size is None: k_transposed = K.permute_dimensions(pre_k, [0, 2, 3, 1]) else: # Memory-compressed attention described in paper # "Generating Wikipedia by Summarizing Long Sequences" # (https://arxiv.org/pdf/1801.10198.pdf) # It compresses keys and values using 1D-convolution which reduces # the size of Q * K_transposed from roughly seq_len^2 # to convoluted_seq_len^2. If we use strided convolution with # window size = 3 and stride = 3, memory requirements of such # memory-compressed attention will be 9 times smaller than # that of the original version. if self.use_masking: raise NotImplementedError( "Masked memory-compressed attention has not " "been implemented yet") k = K.permute_dimensions(pre_k, [0, 2, 1, 3]) k, v = [ K.reshape( # Step 3: Return the result to its original dimensions # (batch_size, num_heads, seq_len, d_model//heads) K.bias_add( # Step 3: ... and add bias K.conv1d( # Step 2: we "compress" K and V using strided conv K.reshape( # Step 1: we reshape K and V to # (batch + num_heads, seq_len, d_model//heads) item, (-1, K.int_shape(item)[-2], d_model // self.num_heads)), kernel, strides=self.compression_window_size, padding='valid', data_format='channels_last'), bias, data_format='channels_last'), # new shape K.concatenate([ K.shape(item)[:2], [ K.int_shape(item)[2] // self.compression_window_size, d_model // self.num_heads ] ])) for item, kernel, bias in ((k, self.k_conv_kernel, self.k_conv_bias), (v, self.v_conv_kernel, self.v_conv_bias)) ] k_transposed = K.permute_dimensions(k, [0, 1, 3, 2]) # shaping K into (batch_size, num_heads, d_model//heads, seq_len) # for further matrix multiplication sqrt_d = K.constant(np.sqrt(d_model // self.num_heads), dtype=K.floatx()) q_shape = K.int_shape(q) k_t_shape = K.int_shape(k_transposed) v_shape = K.int_shape(v) # before performing batch_dot all tensors are being converted to 3D # shape (batch_size * num_heads, rows, cols) to make sure batch_dot # performs identically on all backends attention_heads = K.reshape( K.batch_dot( self.apply_dropout_if_needed(K.softmax( self.mask_attention_if_needed( K.batch_dot( K.reshape(q, (-1, ) + q_shape[-2:]), K.reshape(k_transposed, (-1, ) + k_t_shape[-2:])) / sqrt_d, padding_mask)), training=training), K.reshape(v, (-1, ) + v_shape[-2:])), (-1, self.num_heads, q_shape[-2], v_shape[-1])) attention_heads_merged = K.reshape( K.permute_dimensions(attention_heads, [0, 2, 1, 3]), (-1, d_model)) attention_out = K.reshape( K.dot(attention_heads_merged, self.output_weights), (-1, out_seq_len, d_model)) return attention_out
def style_preview(content, style, output): width, height = load_img(content).size img_height = height img_width = width target_image = K.constant( preprocess_image(content, height=img_height, width=img_width)) style_image = K.constant( preprocess_image(style, height=img_height, width=img_width)) # Placeholder for our generated image generated_image = K.placeholder((1, img_height, img_width, 3)) # Combine the 3 images into a single batch input_tensor = K.concatenate([target_image, style_image, generated_image], axis=0) stderr.write('Start download the model') # The preview network needs to download to the suitable location model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False, cache_dir=".") stderr.write('Model downloading ending') layers = dict([(layer.name, layer.output) for layer in model.layers]) # Set the content and style feature layers content_layer, style_layers = set_cnn_layers(source=source_paper) # initialize total loss loss = K.variable(0.) # add content loss layer_features = layers[content_layer] target_image_features = layer_features[0, :, :, :] combination_features = layer_features[2, :, :, :] loss += content_weight * content_loss(target_image_features, combination_features) # add style loss for layer_name in style_layers: layer_features = layers[layer_name] style_reference_features = layer_features[1, :, :, :] combination_features = layer_features[2, :, :, :] sl = style_loss(style_reference_features, combination_features, height=img_height, width=img_width) loss += (style_weight / len(style_layers)) * sl # add total variation loss loss += total_variation_weight * total_variation_loss( generated_image, img_height, img_width) # Get the gradients of the generated image wrt the loss grads = K.gradients(loss, generated_image)[0] # Function to fetch the values of the current loss and the current gradients fetch_loss_and_grads = K.function([generated_image], [loss, grads]) evaluator = Evaluator(fetch_loss_and_grads=fetch_loss_and_grads, height=img_height, width=img_width) # Run scipy-based optimization (L-BFGS) over the pixels of the generated image # so as to minimize the neural style loss. # This is our initial state: the target image. # Note that `scipy.optimize.fmin_l_bfgs_b` can only process flat vectors. x = preprocess_image(content, height=img_height, width=img_width) x = x.flatten() for _ in range(iterations): x, _, _ = fmin_l_bfgs_b(evaluator.loss, x, fprime=evaluator.grads, maxfun=20) img = x.copy().reshape((img_height, img_width, 3)) img = deprocess_image(img) Image.fromarray(img).save(output, quality=95)
def __init__(self, bias): super(ReverseBiasLayer, self).__init__() self.bias = K.constant((-1) * bias)