def yolo_head(feats, anchors, num_classes, input_shape):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (box_xy + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
box_wh = box_wh * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
return box_xy, box_wh, box_confidence, box_class_probs
def vgg_from_t7(t7_file, target_layer=None):
'''Extract VGG layers from a Torch .t7 model into a Keras model
e.g. vgg = vgg_from_t7('vgg_normalised.t7', target_layer='relu4_1')
Adapted from https://github.com/jonrei/tf-AdaIN/blob/master/AdaIN.py
Converted caffe->t7 from https://github.com/xunhuang1995/AdaIN-style
'''
t7 = torchfile.load(t7_file, force_8bytes_long=True)
inp = Input(shape=(None, None, 3), name='vgg_input')
x = inp
for idx,module in enumerate(t7.modules):
name = module.name.decode() if module.name is not None else None
if idx == 0:
name = 'preprocess' # VGG 1st layer preprocesses with a 1x1 conv to multiply by 255 and subtract BGR mean as bias
if module._typename == b'nn.SpatialReflectionPadding':
x = Lambda(pad_reflect)(x)
elif module._typename == b'nn.SpatialConvolution':
filters = module.nOutputPlane
kernel_size = module.kH
weight = module.weight.transpose([2,3,1,0])
bias = module.bias
x = Conv2D(filters, kernel_size, padding='valid', activation=None, name=name,
kernel_initializer=lambda shape: K.constant(weight, shape=shape),
bias_initializer=lambda shape: K.constant(bias, shape=shape),
trainable=False)(x)
elif module._typename == b'nn.ReLU':
x = Activation('relu', name=name)(x)
elif module._typename == b'nn.SpatialMaxPooling':
x = MaxPooling2D(padding='same', name=name)(x)
# elif module._typename == b'nn.SpatialUpSamplingNearest': # Not needed for VGG
# x = Upsampling2D(name=name)(x)
else:
raise NotImplementedError(module._typename)
if name == target_layer:
# print("Reached target layer", target_layer)
break
# Hook it up
model = Model(inputs=inp, outputs=x)
return model
def call(self, x, mask=None):
if mask is not None:
mask = K.cast(mask, K.floatx())
mask = K.expand_dims(mask, axis=-1)
s = K.sum(mask, axis=1)
if K.equal(s, K.zeros_like(s)) is None:
return K.mean(x, axis=1)
else:
return K.cast(K.sum(x * mask, axis=1) / (K.sqrt(s) + K.constant(1e-10, dtype=K.floatx())), K.floatx())
else:
print (x)
return K.mean(x, axis=1)
def softmax(x, axis, mask=None):
if mask is None:
mask = K.constant(True)
mask = K.cast(mask, K.floatx())
if K.ndim(x) is K.ndim(mask) + 1:
mask = K.expand_dims(mask)
m = K.max(x, axis=axis, keepdims=True)
e = K.exp(x - m) * mask
s = K.sum(e, axis=axis, keepdims=True)
s += K.cast(K.cast(s < K.epsilon(), K.floatx()) * K.epsilon(), K.floatx())
return e / s
def result(input):
emb_c = emb_layer(input)
conv_results_list = []
for cl in conv2d_layers:
conv_results_list.append(cl(emb_c))
emb_c = Lambda(lambda x: K.concatenate(x, axis=3))(conv_results_list)
emb_c = Lambda(lambda x: K.max(x, axis=2))(emb_c)
if highway_on_top:
sigmoid_gate = dense1(emb_c)
sigmoid_gate = Activation('sigmoid')(sigmoid_gate)
deeper_units = dense2(emb_c)
emb_c = Add()([Multiply()([sigmoid_gate, deeper_units]),
Multiply()([Lambda(lambda x: K.constant(1., shape=K.shape(x)) - x)(sigmoid_gate), emb_c])])
emb_c = Activation('relu')(emb_c)
return emb_c
def eval(outputs, anchors, num_classes, image_shape,
max_boxes=20, score_threshold=.6, iou_threshold=.5):
'''Evaluate the YOLO model on given input and return filtered boxes'''
num_layers = len(outputs)
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [
[3, 4, 5], [1, 2, 3]]
input_shape = K.shape(outputs[0])[1:3] * 32
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = boxes_and_scores(outputs[l],
anchors[anchor_mask[l]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use Keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(
class_boxes, class_box_scores, max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
broadcast_shape = [1] * len(input_shape)
broadcast_shape[self.axis] = input_shape[self.axis]
broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape)
broadcast_moving_variance = K.reshape(self.moving_variance,
broadcast_shape)
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
broadcast_beta = K.reshape(self.beta, broadcast_shape)
invstd = (K.ones(shape=broadcast_shape,
dtype='float32')
/ K.sqrt(broadcast_moving_variance
+ K.constant(self.epsilon,
dtype='float32')))
return((inputs - broadcast_moving_mean)
* invstd
* broadcast_gamma
+ broadcast_beta)
def weighted_binary_crossentropy(y_true, y_pred):
class_loglosses = K.mean(K.binary_crossentropy(y_true, y_pred), axis=[0, 1, 2])
return K.sum(class_loglosses * K.constant(class_weights))
num_nodes_min_max_tr,
theta, True)
A = []
Y = []
for index in range(len(input_graphs)):
A.append(networkx.convert_matrix.to_numpy_array(input_graphs[index]))
Y.append(networkx.convert_matrix.to_numpy_array(target_graphs[index]))
print(A)
print(Y)
SYM_NORM = True
A_norm = examples.utils.preprocess_adj_numpy(A, SYM_NORM)
num_filters = 2
graph_conv_filters = examples.utils.np.concatenate(
[A_norm, examples.utils.np.matmul(A_norm, A_norm)], axis=0)
graph_conv_filters = K.constant(graph_conv_filters)
model = Sequential()
model.add(
GraphCNN(16,
num_filters,
graph_conv_filters,
input_shape=(X.shape[1], ),
activation='elu',
kernel_regularizer=l2(5e-4)))
model.add(Dropout(0.2))
model.add(
GraphCNN(Y.shape[1],
num_filters,
graph_conv_filters,
activation='elu',
def log_normals_loss(y_true, y_pred):
y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30)
y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30)
#compute normals with convolution approach
# (http://answers.opencv.org/question/82453/calculate-surface-normals-from-depth-image-using-neighboring-pixels-cross-product/)
config, unparsed = get_config()
batch_size = config.batch_size
#y_true_clipped = K.clip(y_true, K.epsilon(), None)#40.0)
#y_pred_clipped = K.clip(y_pred, K.epsilon(), None)
y_true_clipped = y_true
y_pred_clipped = y_pred
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
w_x = K.variable(np.array([[-1., 0., 1.],
[-1., 0., 1.],
[-1., 0., 1.]]).reshape(3, 3, 1, 1))
w_y = K.variable(np.array([[-1., -1., -1.],
[0., 0., 0.],
[1., 1., 1.]]).reshape(3, 3, 1, 1))
#dzdx = K.conv2d(K.exp(y_true_clipped), w_x, padding='same')
#dzdy = K.conv2d(K.exp(y_true_clipped), w_y, padding='same')
dzdx = K.conv2d(y_true_clipped, w_x, padding='same')
dzdy = K.conv2d(y_true_clipped, w_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
#dzdx_pred = K.conv2d(K.exp(y_pred_clipped), w_x, padding='same')
#dzdy_pred = K.conv2d(K.exp(y_pred_clipped), w_y, padding='same')
dzdx_pred = K.conv2d(y_pred_clipped, w_x, padding='same')
dzdy_pred = K.conv2d(y_pred_clipped, w_y, padding='same')
mag_norm_pred_x = K.pow(dzdx_pred,2) + 1.0
mag_norm_pred_x = K.sqrt(mag_norm_pred_x)
mag_norm_pred_y = K.pow(dzdy_pred, 2) + 1.0
mag_norm_pred_y = K.sqrt(mag_norm_pred_y)
grad_x = K.concatenate(tensors=[K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x,
K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x, dzdx_pred/ mag_norm_pred_x],axis=-1)
grad_y = K.concatenate(tensors=[K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y,
K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y, dzdy_pred/ mag_norm_pred_y],axis=-1)
first_log = K.log(y_pred_clipped + 1.)
second_log = K.log(y_true_clipped + 1.)
log_term = K.sqrt(K.mean(K.square(first_log - second_log), axis=-1) + 0.00001)
dot_term_x = K.sum(normals[:,:,:,:] * grad_x[:,:,:,:], axis=-1, keepdims=True)
dot_term_y = K.sum(normals[:,:,:,:] * grad_y[:,:,:,:], axis=-1, keepdims=True)
#dot_term_x = K.mean(K.sum(normals[:, :, :, :] * grad_x[:, :, :, :], axis=-1, keepdims=True), axis=-1)
#dot_term_y = K.mean(K.sum(normals[:, :, :, :] * grad_y[:, :, :, :], axis=-1, keepdims=True), axis=-1)
dot_term_x = tf.Print(dot_term_x, [dot_term_x], message='dot_term_x', summarize=30)
dot_term_y = tf.Print(dot_term_y, [dot_term_y], message='dot_term_y', summarize=30)
#commentare per vecchia versione
sc_inv_term = K.square(K.mean((first_log - second_log), axis=-1))
norm_term = K.mean(K.square(dot_term_x), axis=-1) + K.mean(K.square(dot_term_y), axis=-1)
diff_x = dzdx_pred - dzdx
diff_y = dzdy_pred - dzdy
grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1)
loss = log_term - (0.5 * sc_inv_term) + norm_term #+ grad_loss
#loss = log_term + K.square(dot_term_x) + K.square(dot_term_y)
return loss
def custom_mod(x, k=K.constant(np.pi), name="custom_mod"):
""" Custom mod function """
y = Lambda(lambda x: K.tf.math.floormod(x - k, 2*k) - k, name=name)(x)
return y
def ccc_loss(gold, pred):
# input (num_batches, seq_len, 1)
ccc_loss = K.constant(1.) - ccc(gold, pred)
return ccc_loss
def call(self, x, mask=None):
'''
Return an anchor box tensor based on the shape of the input tensor.
The logic implemented here is identical to the logic in the module `ssd_box_encode_decode_utils.py`.
Note that this tensor does not participate in any graph computations at runtime. It is being created
as a constant once during graph creation and is just being output along with the rest of the model output
during runtime. Because of this, all logic is implemented as Numpy array operations and it is sufficient
to convert the resulting Numpy array into a Keras tensor at the very end before outputting it.
Arguments:
x (tensor): 4D tensor of shape `(batch, channels, height, width)` if `dim_ordering = 'th'`
or `(batch, height, width, channels)` if `dim_ordering = 'tf'`. The input for this
layer must be the output of the localization predictor layer.
'''
# Compute box width and height for each aspect ratio
# The shorter side of the image will be used to compute `w` and `h` using `scale` and `aspect_ratios`.
self.aspect_ratios = np.sort(self.aspect_ratios)
size = min(self.img_height, self.img_width)
# Compute the box widths and and heights for all aspect ratios
wh_list = []
for ar in self.aspect_ratios:
if (ar == 1) & self.two_boxes_for_ar1:
# Compute the regular default box for aspect ratio 1 and...
w = self.this_scale * size * np.sqrt(ar)
h = self.this_scale * size / np.sqrt(ar)
wh_list.append((w,h))
# ...also compute one slightly larger version using the geometric mean of this scale value and the next
w = np.sqrt(self.this_scale * self.next_scale) * size * np.sqrt(ar)
h = np.sqrt(self.this_scale * self.next_scale) * size / np.sqrt(ar)
wh_list.append((w,h))
else:
w = self.this_scale * size * np.sqrt(ar)
h = self.this_scale * size / np.sqrt(ar)
wh_list.append((w,h))
wh_list = np.array(wh_list)
# We need the shape of the input tensor
if K.image_dim_ordering() == 'tf':
batch_size, feature_map_height, feature_map_width, feature_map_channels = x._keras_shape
else: # Not yet relevant since TensorFlow is the only supported backend right now, but it can't harm to have this in here for the future
batch_size, feature_map_channels, feature_map_height, feature_map_width = x._keras_shape
# Compute the grid of box center points. They are identical for all aspect ratios
cell_height = self.img_height / feature_map_height
cell_width = self.img_width / feature_map_width
cx = np.linspace(cell_width/2, self.img_width-cell_width/2, feature_map_width)
cy = np.linspace(cell_height/2, self.img_height-cell_height/2, feature_map_height)
cx_grid, cy_grid = np.meshgrid(cx, cy)
cx_grid = np.expand_dims(cx_grid, -1) # This is necessary for np.tile() to do what we want further down
cy_grid = np.expand_dims(cy_grid, -1) # This is necessary for np.tile() to do what we want further down
# Create a 4D tensor template of shape `(feature_map_height, feature_map_width, n_boxes, 4)`
# where the last dimension will contain `(cx, cy, w, h)`
boxes_tensor = np.zeros((feature_map_height, feature_map_width, self.n_boxes, 4))
boxes_tensor[:, :, :, 0] = np.tile(cx_grid, (1, 1, self.n_boxes)) # Set cx
boxes_tensor[:, :, :, 1] = np.tile(cy_grid, (1, 1, self.n_boxes)) # Set cy
boxes_tensor[:, :, :, 2] = wh_list[:, 0] # Set w
boxes_tensor[:, :, :, 3] = wh_list[:, 1] # Set h
# Convert `(cx, cy, w, h)` to `(xmin, xmax, ymin, ymax)`
boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='centroids2minmax')
# If `limit_boxes` is enabled, clip the coordinates to lie within the image boundaries
if self.limit_boxes:
x_coords = boxes_tensor[:,:,:,[0, 1]]
x_coords[x_coords >= self.img_width] = self.img_width - 1
x_coords[x_coords < 0] = 0
boxes_tensor[:,:,:,[0, 1]] = x_coords
y_coords = boxes_tensor[:,:,:,[2, 3]]
y_coords[y_coords >= self.img_height] = self.img_height - 1
y_coords[y_coords < 0] = 0
boxes_tensor[:,:,:,[2, 3]] = y_coords
# `normalize_coords` is enabled, normalize the coordinates to be within [0,1]
if self.normalize_coords:
boxes_tensor[:, :, :, :2] /= self.img_width
boxes_tensor[:, :, :, 2:] /= self.img_height
if self.coords == 'centroids':
# TODO: Implement box limiting directly for `(cx, cy, w, h)` so that we don't have to unnecessarily convert back and forth
# Convert `(xmin, xmax, ymin, ymax)` back to `(cx, cy, w, h)`
boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='minmax2centroids')
# 4: Create a tensor to contain the variances and append it to `boxes_tensor`. This tensor has the same shape
# as `boxes_tensor` and simply contains the same 4 variance values for every position in the last axis.
variances_tensor = np.zeros_like(boxes_tensor) # Has shape `(feature_map_height, feature_map_width, n_boxes, 4)`
variances_tensor += self.variances # Long live broadcasting
# Now `boxes_tensor` becomes a tensor of shape `(feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.concatenate((boxes_tensor, variances_tensor), axis=-1)
# Now prepend one dimension to `boxes_tensor` to account for the batch size and tile it along
# The result will be a 5D tensor of shape `(batch_size, feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.expand_dims(boxes_tensor, axis=0)
boxes_tensor = K.tile(K.constant(boxes_tensor, dtype='float32'), (K.shape(x)[0], 1, 1, 1, 1))
return boxes_tensor
def jaccard_loss(y_true, y_pred, weights):
batch_jaccard_coefs = soft_jaccard(y_true, y_pred, axis=[1, 2])
jaccard_coefs = K.mean(batch_jaccard_coefs, axis=0)
w = K.constant(weights) / sum(weights)
return 1 - K.sum(w * jaccard_coefs)
import numpy as np import tensorflow.spectral as tf from keras import backend as K #from so3_fft import so3_rfft, _setup_wigner x = [[1.1, 2.2], [3.3, 4.4]] x = K.constant(x, dtype='complex64') y = K.eval(x) print(y.shape[0])
def corr(y_true, y_pred):
return K.constant((numpy.corrcoef(y_true.flatten(), y_pred.flatten())[0, 1]) ** 2)
def _build_model(self):
rpn_trainable = self.config.training_mode in ['rpn_only', 'all']
head_trainable = self.config.training_mode in ['head_only', 'all']
# backbone network
backbone_in, backbone_out = self._model_backbone_headless()
# rpn
normalized_rois, rpn_offsets, objects, objects_logit \
= self._nn_rpn(backbone_out, rpn_trainable)
# 学習時のみ損失を計算
if self.config.training:
# 学習時
input_gt_rois = Input(
shape=[None, 4], name="input_gt_rois", dtype='float32')
input_gt_objects = Input(
shape=[None], name="input_gt_objects", dtype='int32')
inputs = [backbone_in, input_gt_rois, input_gt_objects]
losses = []
if rpn_trainable:
# 損失計算
# RPNの損失
rpn_offsets_loss = Lambda(lambda x: loss.rpn_offsets_loss(*x),
name="rpn_offsets_loss")(
[input_gt_rois, input_gt_objects, rpn_offsets])
rpn_objects_loss = Lambda(lambda x: loss.rpn_objects_loss(*x),
name="rpn_objects_loss")(
[input_gt_objects, objects])
losses += [rpn_offsets_loss, rpn_objects_loss]
if head_trainable:
input_gt_boxes = Input(
shape=[None, 4], name="input_gt_boxes", dtype='float32')
input_gt_label_ids = Input(
shape=[None], name="input_gt_label_ids", dtype='int32')
h, w = self.config.image_shape[0], self.config.image_shape[1]
input_gt_masks = Input(
shape=[None, h, w], name="input_gt_masks", dtype='float32')
inputs += [input_gt_boxes, input_gt_label_ids, input_gt_masks]
# 正解データとRoIから評価対象のRoIを絞り込み、それに対応する正解データを得る。
normalized_sample_rois, normalized_sample_gt_offsets, \
sample_gt_labels, sample_gt_masks = \
MaskSubsamplingRoiLayer(self.config,
name='mask_subsampling')(
[normalized_rois, input_gt_boxes,
input_gt_label_ids, input_gt_masks])
# head
head_offsets, labels, labels_logit \
= self._nn_head(backbone_out, normalized_sample_rois)
# ヘッドの損失はModel#compileで損失関数を指定する方法では対応出来ないため、
# Layerとして定義してModel#add_lossで加算する。
head_offsets_loss = Lambda(lambda x:
loss.head_offsets_loss(*x),
name="head_offsets_loss")(
[normalized_sample_gt_offsets, sample_gt_labels,
head_offsets])
head_labels_loss = Lambda(lambda x:
loss.head_labels_loss(*x),
name="head_labels_loss")(
[sample_gt_labels, labels])
# mask
masks = self._nn_mask(backbone_out, normalized_sample_rois)
head_mask_loss = Lambda(lambda x:
loss.head_mask_loss(*x),
name="head_mask_loss")(
[sample_gt_masks, sample_gt_labels, masks])
# 損失
losses += [head_offsets_loss, head_labels_loss, head_mask_loss]
# 出力＝損失
outputs = losses
else:
# 予測時
# head
# head_offsetsは0〜1で正規化された値
head_offsets, labels, _ = self._nn_head(
backbone_out, normalized_rois)
# 候補を絞り込む
bboxes, rois, labels, scores = Lambda(lambda x:
self._nn_squeeze_roi(*x),
name="squeeze_roi")(
[normalized_rois, head_offsets, labels])
# mask
masks = self._nn_mask(backbone_out, rois)
def _squeeze_masks(masks, idx_labels):
dim1 = K.flatten(K.repeat(K.expand_dims(
K.arange(K.shape(masks)[0])), K.shape(masks)[1]))
dim2 = K.tile(K.arange(K.shape(masks)[1]),
[K.shape(masks)[0]])
idx = K.stack([dim1, dim2,
K.cast(K.flatten(labels), tf.int32)], axis=1)
# idx = log.tfprint(idx, "idx:inloop:debug")
squeezed_masks = tf.gather_nd(masks, idx)
squeezed_masks = K.reshape(squeezed_masks,
[K.shape(masks)[0],
K.shape(masks)[1],
K.shape(masks)[3],
K.shape(masks)[4]])
return squeezed_masks
# ラベルに対応するマスクを残す
masks = Lambda(lambda x: _squeeze_masks(x[0],
K.cast(x[1], tf.int32)),
name="squeeze_mask")([masks, labels])
# 予測時は損失不要
# ダミーの損失関数
dummy_loss = Lambda(lambda x: K.constant(0), name="dummy_loss")(
[backbone_in])
losses = [dummy_loss, dummy_loss, dummy_loss,
dummy_loss, dummy_loss, dummy_loss]
inputs = [backbone_in]
outputs = [bboxes, labels, scores, masks, rois,
rpn_offsets, objects]
model = Model(inputs=inputs, outputs=outputs, name='mask_r_cnn')
# Kerasは複数指定した損失の合計をモデル全体の損失として評価してくれる。
# 損失を追加
for output in losses:
model.add_loss(tf.reduce_mean(output, keep_dims=True))
return model, len(outputs)
def _smooth_labels(y_true, label_smoothing):
num_classes = tf.cast(K.shape(y_true)[-1], dtype=K.floatx())
label_smoothing = K.constant(label_smoothing, dtype=K.floatx())
return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes
def drop_last_row(shape, dtype=None): extractor = np.zeros(shape) for i in range(np.min(shape)): extractor[i][i] = 1 return K.constant(extractor)
def extract_last_row(shape, dtype=None): extractor = np.zeros(shape) extractor[-1][-1] = 0 return K.constant(extractor)
def __call__(self, shape, dtype=None):
return K.constant(1 / K.sqrt(16), shape=shape, dtype=dtype)
def yolo_eval(yolo_outputs,
anchors,
num_classes,
image_shape,
max_boxes=20,
score_threshold=.6,
iou_threshold=.5):
# 获得特征层的数量
num_layers = len(yolo_outputs)
# 特征层1对应的anchor是678（13，13）
# 特征层2对应的anchor是345（26，26）
# 特征层3对应的anchor是012（52，52）
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
# 416，416
boxes = []
box_scores = []
# ---------------------------------对每一个特征层的结果进行处理-----------------------
# 对每个特征层进行处理
for l in range(num_layers):
_boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
# 将每个特征层的结果进行堆叠（全部堆叠成一维）
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
# 每一个点对应80种类别的得分值
# 通过每个得分和阈值进行比较，满足要求的就留下，并设置为1
mask = box_scores >= score_threshold
# 限制每个图片最大的边框数（20）
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# 取出所有box_scores >= score_threshold的框，和成绩
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
# 非极大抑制，去掉box重合程度高的那一些
nms_index = tf.image.non_max_suppression(class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
# 获取非极大抑制后的结果
# 下列三个分别是
# 框的位置，得分与种类
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def __init__(self, switch_matrix, pool_size):
super(MaxUnPooling2D, self).__init__()
self.switch_matrix = K.constant(switch_matrix)
self.pool_size = pool_size
def mean_squared_error(y_true, y_pred):
if not K.is_tensor(y_pred):
y_pred = K.constant(y_pred)
y_true = K.cast(y_true, y_pred.dtype)
return K.mean(K.square(y_pred - y_true), axis=-1)
def call(self, x, mask=None):
'''
Return an anchor box tensor based on the shape of the input tensor.
The logic implemented here is identical to the logic in the module `ssd_box_encode_decode_utils.py`.
Note that this tensor does not participate in any graph computations at runtime. It is being created
as a constant once during graph creation and is just being output along with the rest of the model output
during runtime. Because of this, all logic is implemented as Numpy array operations and it is sufficient
to convert the resulting Numpy array into a Keras tensor at the very end before outputting it.
Arguments:
x (tensor): 4D tensor of shape `(batch, channels, height, width)` if `dim_ordering = 'th'`
or `(batch, height, width, channels)` if `dim_ordering = 'tf'`. The input for this
layer must be the output of the localization predictor layer.
'''
# Compute box width and height for each aspect ratio
# The shorter side of the image will be used to compute `w` and `h` using `scale` and `aspect_ratios`.
size = min(self.img_height, self.img_width)
# Compute the box widths and and heights for all aspect ratios
wh_list = []
for ar in self.aspect_ratios:
if (ar == 1):
# Compute the regular anchor box for aspect ratio 1.
box_height = box_width = self.this_scale * size
wh_list.append((box_width, box_height))
if self.two_boxes_for_ar1:
# Compute one slightly larger version using the geometric mean of this scale value and the next.
box_height = box_width = np.sqrt(self.this_scale * self.next_scale) * size
wh_list.append((box_width, box_height))
else:
box_height = self.this_scale * size / np.sqrt(ar)
box_width = self.this_scale * size * np.sqrt(ar)
wh_list.append((box_width, box_height))
wh_list = np.array(wh_list)
# We need the shape of the input tensor
if K.image_dim_ordering() == 'tf':
batch_size, feature_map_height, feature_map_width, feature_map_channels = x._keras_shape
else: # Not yet relevant since TensorFlow is the only supported backend right now, but it can't harm to have this in here for the future
batch_size, feature_map_channels, feature_map_height, feature_map_width = x._keras_shape
# Compute the grid of box center points. They are identical for all aspect ratios.
# Compute the step sizes, i.e. how far apart the anchor box center points will be vertically and horizontally.
if (self.this_steps is None):
step_height = self.img_height / feature_map_height
step_width = self.img_width / feature_map_width
else:
if isinstance(self.this_steps, (list, tuple)) and (len(self.this_steps) == 2):
step_height = self.this_steps[0]
step_width = self.this_steps[1]
elif isinstance(self.this_steps, (int, float)):
step_height = self.this_steps
step_width = self.this_steps
# Compute the offsets, i.e. at what pixel values the first anchor box center point will be from the top and from the left of the image.
if (self.this_offsets is None):
offset_height = 0.5
offset_width = 0.5
else:
if isinstance(self.this_offsets, (list, tuple)) and (len(self.this_offsets) == 2):
offset_height = self.this_offsets[0]
offset_width = self.this_offsets[1]
elif isinstance(self.this_offsets, (int, float)):
offset_height = self.this_offsets
offset_width = self.this_offsets
# Now that we have the offsets and step sizes, compute the grid of anchor box center points.
cy = np.linspace(offset_height * step_height, (offset_height + feature_map_height - 1) * step_height,
feature_map_height)
cx = np.linspace(offset_width * step_width, (offset_width + feature_map_width - 1) * step_width,
feature_map_width)
cx_grid, cy_grid = np.meshgrid(cx, cy)
cx_grid = np.expand_dims(cx_grid, -1) # This is necessary for np.tile() to do what we want further down
cy_grid = np.expand_dims(cy_grid, -1) # This is necessary for np.tile() to do what we want further down
# Create a 4D tensor template of shape `(feature_map_height, feature_map_width, n_boxes, 4)`
# where the last dimension will contain `(cx, cy, w, h)`
boxes_tensor = np.zeros((feature_map_height, feature_map_width, self.n_boxes, 4))
boxes_tensor[:, :, :, 0] = np.tile(cx_grid, (1, 1, self.n_boxes)) # Set cx
boxes_tensor[:, :, :, 1] = np.tile(cy_grid, (1, 1, self.n_boxes)) # Set cy
boxes_tensor[:, :, :, 2] = wh_list[:, 0] # Set w
boxes_tensor[:, :, :, 3] = wh_list[:, 1] # Set h
# Convert `(cx, cy, w, h)` to `(xmin, xmax, ymin, ymax)`
boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='centroids2corners')
# If `clip_boxes` is enabled, clip the coordinates to lie within the image boundaries
if self.clip_boxes:
x_coords = boxes_tensor[:, :, :, [0, 2]]
x_coords[x_coords >= self.img_width] = self.img_width - 1
x_coords[x_coords < 0] = 0
boxes_tensor[:, :, :, [0, 2]] = x_coords
y_coords = boxes_tensor[:, :, :, [1, 3]]
y_coords[y_coords >= self.img_height] = self.img_height - 1
y_coords[y_coords < 0] = 0
boxes_tensor[:, :, :, [1, 3]] = y_coords
# If `normalize_coords` is enabled, normalize the coordinates to be within [0,1]
if self.normalize_coords:
boxes_tensor[:, :, :, [0, 2]] /= self.img_width
boxes_tensor[:, :, :, [1, 3]] /= self.img_height
# TODO: Implement box limiting directly for `(cx, cy, w, h)` so that we don't have to unnecessarily convert back and forth.
if self.coords == 'centroids':
# Convert `(xmin, ymin, xmax, ymax)` back to `(cx, cy, w, h)`.
boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='corners2centroids')
elif self.coords == 'minmax':
# Convert `(xmin, ymin, xmax, ymax)` to `(xmin, xmax, ymin, ymax).
boxes_tensor = convert_coordinates(boxes_tensor, start_index=0, conversion='corners2minmax')
# Create a tensor to contain the variances and append it to `boxes_tensor`. This tensor has the same shape
# as `boxes_tensor` and simply contains the same 4 variance values for every position in the last axis.
variances_tensor = np.zeros_like(
boxes_tensor) # Has shape `(feature_map_height, feature_map_width, n_boxes, 4)`
variances_tensor += self.variances # Long live broadcasting
# Now `boxes_tensor` becomes a tensor of shape `(feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.concatenate((boxes_tensor, variances_tensor), axis=-1)
# Now prepend one dimension to `boxes_tensor` to account for the batch size and tile it along
# The result will be a 5D tensor of shape `(batch_size, feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.expand_dims(boxes_tensor, axis=0)
boxes_tensor = K.tile(K.constant(boxes_tensor, dtype='float32'), (K.shape(x)[0], 1, 1, 1, 1))
return boxes_tensor
def mean_absolute_error_penalty(y_true, y_pred, penalty):
penalty = K.constant(penalty, dtype='float32')
return K.mean(penalty * K.abs(y_pred - y_true), axis=-1)
def call(self, inputs, training=None):
input_shape = K.int_shape(inputs)
broadcast_shape = [1] * len(input_shape)
broadcast_shape[self.axis] = input_shape[self.axis]
broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape)
broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape)
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
broadcast_beta = K.reshape(self.beta, broadcast_shape)
invstd = K.ones (shape=broadcast_shape, dtype='float32') / K.sqrt(broadcast_moving_variance + K.constant(self.epsilon, dtype='float32'))
return (inputs - broadcast_moving_mean) * invstd * broadcast_gamma + broadcast_beta
def fit(self, Xs, ys, dataset_weights=None):
self.random_state = check_random_state(self.random_state)
n_datasets = len(Xs)
n_samples = sum(X.shape[0] for X in Xs)
n_features = Xs[0].shape[1]
# Data
sizes = np.array([y.shape[1] for y in ys])
limits = [0] + np.cumsum(sizes).tolist()
n_targets = limits[-1]
if self.n_components == 'auto':
n_components = n_targets
else:
n_components = self.n_components
self.slices_ = []
for iter in range(n_datasets):
self.slices_.append(np.array([limits[iter], limits[iter + 1]]))
self.slices_ = tuple(self.slices_)
if dataset_weights is None:
dataset_weights = [1.] * n_datasets
dataset_weights /= np.sum(dataset_weights) / n_datasets
# dataset_weights = np.array(dataset_weights) * np.sqrt([X.shape[0] for X in Xs])
padded_ys = []
masks = []
Xs = [X.copy() for X in Xs]
for y, this_slice in zip(ys, self.slices_):
padded_y = np.zeros((y.shape[0], n_targets))
mask = np.zeros((y.shape[0], n_targets), dtype='bool')
mask[:, this_slice[0]:this_slice[1]] = 1
padded_y[:, this_slice[0]:this_slice[1]] = y
padded_ys.append(padded_y)
masks.append(mask)
if self.use_generator:
our_generator = generator(Xs,
padded_ys,
masks,
batch_size=self.batch_size,
dataset_weights=dataset_weights,
random_state=self.random_state)
if self.batch_size is not None:
steps_per_epoch = n_samples / self.batch_size
else:
steps_per_epoch = n_datasets
else:
Xs_cat = np.concatenate(Xs, axis=0)
padded_ys_cat = np.concatenate(padded_ys, axis=0)
masks_cat = np.concatenate(masks, axis=0)
sample_weight = np.concatenate(
[[dataset_weight * n_samples / X.shape[0] / n_datasets] *
X.shape[0] for dataset_weight, X in zip(dataset_weights, Xs)])
if self.batch_size is None:
batch_size = n_samples
else:
batch_size = self.batch_size
# Model
if self.intercept_init is not None:
supervised_bias_initializer = \
lambda shape: K.constant(self.intercept_init)
else:
supervised_bias_initializer = 'zeros'
if self.coef_init is not None:
U, S, VT = svd(self.coef_init, full_matrices=False)
latent_init = U[:, :n_components]
latent_init *= S[:n_components]
supervised_init = VT[:n_components]
if n_components > latent_init.shape[1]:
latent_init = np.concatenate([
latent_init,
np.zeros((latent_init.shape[0],
n_components - latent_init.shape[1]))
],
axis=1)
supervised_init = np.concatenate([
supervised_init,
np.zeros((n_components - supervised_init.shape[0],
supervised_init.shape[1]))
],
axis=0)
supervised_kernel_initializer = \
lambda shape: K.constant(supervised_init)
latent_kernel_initializer = \
lambda shape: K.constant(latent_init)
else:
supervised_kernel_initializer = 'glorot_uniform'
latent_kernel_initializer = 'glorot_uniform'
data = Input(shape=(n_features, ), name='data', dtype='float32')
mask = Input(shape=(n_targets, ), name='mask', dtype='bool')
dropout_data = Dropout(rate=self.input_dropout_rate)(data)
if n_components is not None:
latent = Dense(n_components,
name='latent',
use_bias=False,
kernel_initializer=latent_kernel_initializer,
kernel_regularizer=l2(self.alpha))(dropout_data)
latent = Dropout(rate=self.latent_dropout_rate)(latent)
else:
latent = dropout_data
logits = Dense(n_targets,
use_bias=True,
name='supervised',
kernel_initializer=supervised_kernel_initializer,
bias_initializer=supervised_bias_initializer,
kernel_regularizer=l2(self.alpha))(latent)
prediction = PartialSoftmax()([logits, mask])
self.model_ = Model(inputs=[data, mask], outputs=prediction)
if self.optimizer == 'adam':
optimizer = Adam
elif self.optimizer == 'sgd':
optimizer = SGD
elif self.optimizer == 'rmsprop':
optimizer = RMSprop
else:
raise ValueError('Wrong optimizer')
self.model_.compile(optimizer(self.step_size),
loss='categorical_crossentropy',
metrics=['accuracy'])
schedule = lambda i: 1e-3 if i < self.max_iter // 2 else 1e-4
scheduler = LearningRateScheduler(schedule)
callbacks = [scheduler]
if self.latent_sparsity is not None:
callbacks.append(L1ProjCallback(radius=self.latent_sparsity))
sess = tf.Session(
config=tf.ConfigProto(device_count={'CPU': self.n_jobs},
inter_op_parallelism_threads=self.n_jobs,
intra_op_parallelism_threads=self.n_jobs,
use_per_session_threads=True))
K.set_session(sess)
if self.use_generator:
self.model_.fit_generator(our_generator,
steps_per_epoch=steps_per_epoch,
verbose=2,
epochs=self.max_iter,
callbacks=callbacks)
else:
self.model_.fit([Xs_cat, masks_cat],
padded_ys_cat,
sample_weight=sample_weight,
batch_size=batch_size,
verbose=2,
epochs=self.max_iter,
callbacks=callbacks)
supervised, self.intercept_ = self.model_.get_layer(
'supervised').get_weights()
if self.n_components is not None:
latent = self.model_.get_layer('latent').get_weights()[0]
self.coef_ = latent.dot(supervised)
else:
self.coef_ = supervised
img = vgg19.preprocess_input(img)
return img
def deprocess_image(x):
x[:, :, 0] += 103.939
x[:, :, 1] += 116.779
x[:, :, 2] += 123.68
# BGR -> RGB
x = x[:, :, ::-1]
x = np.clip(x, 0, 255).astype('uint8')
return x
# 读取VGG19网络并应用到我们的图片中
target_image = K.constant(preprocess_image(target_image_path))
style_reference_image = K.constant(preprocess_image(style_reference_image_path))
# 生成的图像
combination_image = K.placeholder((1, img_height, img_width, 3))
input_tensor = K.concatenate([target_image,
style_reference_image,
combination_image], axis=0)
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
print('Model loaded.')
def __init__(self,
wordvec=base + 'data/wordvec.txt',
window=1,
maxlen=100,
filter=250,
pe=False,
drop_emb=.5,
drop_conv=.5,
drop_sa=.25,
drop_res=0,
drop_dense=0,
feature=0,
n_layers=1,
sa_scale=0,
residue=False,
dense_layers=[],
pad_position=['pre', 'pre'],
n_relation=4,
n_clause=2,
torel=None):
if type(dense_layers) is int:
dense_layers = [dense_layers]
self.__dict__.update(locals())
if torel is not None:
self.n_relation = n_relation = len(torel)
self.wordvec = wordvec = word2vec.load(wordvec)
vs, es = wordvec.vectors.shape
# Input Layers
inputs = [
Input(shape=(maxlen, ), name='clause%d' % i)
for i in range(n_clause)
]
if self.feature:
feature = [Input(shape=(feature, ), name='feature')]
else:
feature = []
# Hidden Layers
embed = Embedding(vs,
es,
input_length=maxlen,
weights=[wordvec.vectors],
name='embed')
if sa_scale:
attn = Lambda(scaled_attention(sa_scale), name='attention')
drop_sa = Dropout(drop_sa, name='drop_sa')
if not hasattr(filter, '__getitem__'):
filter = (filter, ) * n_layers
if not hasattr(drop_conv, '__getitem__'):
drop_conv = (drop_conv, ) * n_layers
if not hasattr(drop_res, '__getitem__'):
drop_res = (drop_res, ) * n_layers
if not hasattr(drop_dense, '__getitem__'):
drop_dense = (drop_dense, ) * len(dense_layers)
if self.pe:
dim = es
pe_i, pe_p = np.meshgrid(range(dim), range(maxlen))
pe = np.sin(2 * np.pi * pe_p / maxlen * pe_i / dim)
#pe = np.sin(pe_p/10000**(2*pe_i/dim))
#pe /= wordvec.vectors.max()*10
pe_const = K.constant(pe, name='pe')
pe = Lambda(lambda x: x + pe_const, name='pos_enc')
if residue:
reduce = [[
Dense(filter[j], name='reduce%d-%d' % (i, j))
for j in range(n_layers)
] for i in range(n_clause)]
add = Add(name='add')
drop_res = [
Dropout(drop_res[j], name='drop_res-%d' % j)
for j in range(n_layers)
]
conv = [[
Conv1D(filter[j],
window,
activation='relu',
name='conv%d-%d' % (i, j)) for j in range(n_layers)
] for i in range(n_clause)]
drop_conv = [
Dropout(drop_conv[j], name='drop_conv-%d' % j)
for j in range(n_layers)
]
pool = MaxPooling1D(maxlen, padding='same', name='pool')
drop_emb = Dropout(drop_emb, name='drop_emb')
flatten = Flatten(name='flatten')
dense_layers = [
Dense(n, activation='relu', name='dense%d' % i)
for i, n in enumerate(dense_layers)
]
drop_dense = [
Dropout(dp, name='drop_dense-%d' % j)
for j, dp in enumerate(drop_dense)
]
dense_out = Dense(n_relation, activation='softmax', name='dense_out')
concat = Concatenate(name='concat')
# Forward
h = [embed(clause) for clause in inputs]
if self.pe:
h = [pe(hi) for hi in h]
h = [drop_emb(hi) for hi in h]
if sa_scale:
h = [attn(hi) for hi in h]
h = [drop_sa(hi) for hi in h]
for j in range(n_layers):
if residue:
r = [reduce[i][j](h[i]) for i in range(n_clause)]
r = [drop_res[j](ri) for ri in r]
h = [conv[i][j](h[i]) for i in range(n_clause)]
h = [drop_conv[j](hi) for hi in h]
if residue:
h = [add([h[i], r[i]]) for i in range(n_clause)]
h = [pool(hi) for hi in h]
h = [flatten(hi) for hi in h]
h = concat(h + feature)
for dense, drop in zip(dense_layers, drop_dense):
h = drop(dense(h))
output = dense_out(h)
super().__init__(inputs=inputs + feature, outputs=output)
self.init_weights = self.get_weights()
def _build_layers_v2(self, input_dict, num_outputs, options):
TRANSFORMER_SIMPLICIAL_DIM = options["custom_options"][
"transformer_simplicial_model_dim"]
TRANSFORMER_MODEL_DIM = options["custom_options"][
"transformer_model_dim"]
TRANSFORMER_NUM_HEADS = options["custom_options"][
"transformer_num_heads"]
TRANSFORMER_DEPTH = options["custom_options"]["transformer_depth"]
CONV_PADDING = options["custom_options"]["conv_padding"]
NUM_VIRTUAL_ENTITIES = options["custom_options"][
"num_virtual_entities"]
# For detailed comments see the base agent
inputs = input_dict["obs"]
sess = tf.get_default_session()
K.set_session(sess)
attention_layer = MultiHeadSelfAttentionZambaldi(
name='self_attention',
num_heads=TRANSFORMER_NUM_HEADS,
use_masking=False,
dropout=0,
compression_window_size=None,
num_virtual_entities=NUM_VIRTUAL_ENTITIES)
attention_layer_2simplex = MultiHeadSelfAttentionSimplicial(
name='self_2attention',
num_heads=1,
d_simp_model=TRANSFORMER_SIMPLICIAL_DIM,
use_masking=False,
dropout=0,
compression_window_size=None,
num_virtual_entities=NUM_VIRTUAL_ENTITIES)
dense_layer1 = layers.Dense(TRANSFORMER_MODEL_DIM, activation='relu')
dense_layer2 = layers.Dense(TRANSFORMER_MODEL_DIM)
def transformer_block(input):
a = LayerNormalization()(input)
a1 = attention_layer(
a
) # a1 = attention(h) has shape -1, seq_len, TRANSFORMER_MODEL_DIM
a2 = attention_layer_2simplex(
a) # shape -1, seq_len, TRANSFORMER_SIMPLICIAL_DIM
a2 = LayerNormalization()(a2)
ac = Concatenate()(
[a1, a2]
) # shape -1, seq_len, TRANSFORMER_MODEL_DIM + TRANSFORMER_SIMPLICIAL_DIM
b = dense_layer1(ac)
b2 = dense_layer2(b) # b = ff(ac)
r = layers.Add()([input, b2])
Hprime = LayerNormalization()(r)
return Hprime
# CONVOLUTIONS ------
#
x = layers.Lambda(lambda x: x / 255)(inputs) # rescale RGB to [0,1]
x = layers.Conv2D(12, (2, 2), activation='relu',
padding=CONV_PADDING)(x)
x = layers.Conv2D(24, (2, 2), activation='relu', padding=CONV_PADDING)(
x) # output shape -1, num_rows, num_cols, 62
x = layers.Dense(
TRANSFORMER_MODEL_DIM - 2, activation=None, use_bias=False
)(x) # output shape -1, num_rows, num_cols, TRANSFORMER_MODEL_DIM-2
# POSITION EMBEDDING -----
#
num_rows, num_cols, d_model = x.get_shape().as_list()[-3:]
ps = np.zeros([num_rows, num_cols, 2],
dtype=K.floatx()) # shape (12,13,2)
for ty in range(num_rows):
for tx in range(num_cols):
ps[ty, tx, :] = [(2 / (num_rows - 1)) * ty - 1,
(2 / (num_cols - 1)) * tx - 1]
ps_expand = K.expand_dims(K.constant(ps),
axis=0) # shape (1,num_rows,num_cols,2)
ps_tiled = K.tile(
ps_expand,
[K.shape(x)[0], 1, 1, 1]) # shape (None,num_rows,num_cols,2)
# (None,num_rows,num_cols,62) concatenate with (None,num_rows,num_cols,2)
# to get (None,num_rows,num_cols,TRANSFORMER_MODEL_DIM)
x = Concatenate(axis=3)([x, ps_tiled])
x = layers.Reshape(
(num_rows * num_cols,
d_model + 2))(x) # shape (None, num_rows*num_cols,d_model+2)
# NOTE: the batch dimension is preserved by reshape, see https://www.tensorflow.org/api_docs/python/tf/keras/layers/Reshape
# We now add some virtual entities, which are initialised randomly
tokens = np.arange(NUM_VIRTUAL_ENTITIES).reshape(
(1, NUM_VIRTUAL_ENTITIES)) # [[0,1,2,...,NUM_VIRTUAL_ENTITIES]]
tokens = K.constant(tokens)
ve = layers.Embedding(
input_dim=NUM_VIRTUAL_ENTITIES, output_dim=d_model + 2)(
tokens) # shape (1,NUM_VIRTUAL_ENTITIES,d_model+2)
ve_tiled = K.tile(ve, [K.shape(x)[0], 1, 1])
x = Concatenate(axis=1)([x, ve_tiled])
# TRANSFORMER -----
for i in range(TRANSFORMER_DEPTH):
x = transformer_block(x)
# The output of the simplicial Transformer includes the virtual entities,
# which we now want to remove. The current tensor is of shape
# (None,num_rows*num_cols+NUM_VIRTUAL_ENTITIES,TRANSFORMER_MODEL_DIM)
x = x[:, :-NUM_VIRTUAL_ENTITIES, :]
# MAX-POOLING -----
# from p.4 "The E~ matrix, with shape Nxf is reudced to an f-dimensional vector by max-pooling
# over the entity dimension. This pooled vector is then passed to a small MLP..."
num_entities, d_model = x.get_shape().as_list()[-2:]
x = layers.MaxPooling1D(pool_size=num_entities)(x)
x = layers.Flatten()(x)
# FULLY-CONNECTED LAYERS ----
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
output_tensor = layers.Dense(4)(x) # final output is logits
return output_tensor, x
content_weight = 0.025 #Weight of image content
total_variation_weight = 10 ** -4 #Weight of image variation
out_h = 400 #Output image height in pixels
iterations = 20 #Number of transfer interations
'''
Prepare images, model, and loss function.
'''
#Calculate width to scale output image to a height of out_h.
w, h = load_img(target_file).size
out_w = int(w * out_h / h)
#Load pre-processed images and prepare placeholder.
target = K.constant(sdu.preprocess_image(target_file, out_h, out_w))
style = K.constant(sdu.preprocess_image(style_file, out_h, out_w))
combined = K.placeholder((1, out_h, out_w, 3))
#Combine images into a single tensor.
input_tensor = K.concatenate([target, style, combined], axis = 0)
#Load VGG19 network with ImageNet weights.
model = vgg19.VGG19(input_tensor = input_tensor, weights = "imagenet",
include_top = False)
#Define general loss function.
weights = [content_weight, style_weight, total_variation_weight]
loss = sdu.general_loss(model, weights, combined, out_h, out_w)
#Get gradients of the generated image.
F_u = Dropout(0.1)(F_u)
F_u = WeightedAvgOverTime()(F_u)
U = Add()([U, F_u])
#F_b = Reshape((feedback_b.shape[1],))(Embedding(book_num, feedback_b.shape[1], trainable=False, weights=[feedback_b])(b_input))
#F_b = Embedding(user_num+1, args.emb_dim, embeddings_initializer=Zeros(), embeddings_regularizer=l2(0.00001), mask_zero=True)(F_b)
#F_b = Dropout(0.1)(F_b)
#F_b = WeightedAvgOverTime()(F_b)
#B = Add()([B, F_b])
pred = Dot(axes=-1)([U, B])
pred = Add()([pred, U_bias, B_bias])
pred = Lambda(lambda x: x + K.constant(rate_mean, dtype=K.floatx()))(pred)
else:
pred = Dot(axes=-1)([U, B])
pred = Add()([pred, U_bias, B_bias])
else:
pred = Concatenate()([U, B])
for units in args.dense_units:
pred = Dense(units, activation='selu')(pred)
pred = Dropout(0.1)(pred)
if args.norm_ans:
pred = Dense(1, activation='selu')(pred)
else:
pred = Dense(1)(pred)
pred = Add()([pred, U_bias, B_bias])
def DeepConv1DOptLearnerStaticArchitecture(param_trainable, init_wrapper, smpl_params, input_info, faces, emb_size=1000, input_type="3D_POINTS"):
""" Optimised learner network architecture """
# An embedding layer is required to optimise the parameters
optlearner_input = Input(shape=(1,), name="embedding_index")
# Initialise the embedding layers
emb_layers = init_emb_layers(optlearner_input, emb_size, param_trainable, init_wrapper)
optlearner_params = Concatenate(name="parameter_embedding")(emb_layers)
optlearner_params = Reshape(target_shape=(85,), name="learned_params")(optlearner_params)
print("optlearner parameters shape: " +str(optlearner_params.shape))
#exit(1)
# Ground truth parameters and point cloud are inputs to the model as well
gt_params = Input(shape=(85,), name="gt_params")
gt_pc = Input(shape=(6890, 3), name="gt_pc")
print("gt parameters shape: " + str(gt_params.shape))
print("gt point cloud shape: " + str(gt_pc.shape))
# Compute the true offset (i.e. difference) between the ground truth and learned parameters
pi = K.constant(np.pi)
delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d")([gt_params, optlearner_params])
#delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d_no_mod")([gt_params, optlearner_params])
#delta_d = Lambda(lambda x: K.tf.math.floormod(x - pi, 2*pi) - pi, name="delta_d")(delta_d) # custom modulo 2pi of delta_d
#delta_d = custom_mod(delta_d, pi, name="delta_d") # custom modulo 2pi of delta_d
print("delta_d shape: " + str(delta_d.shape))
#exit(1)
# Calculate the (batched) MSE between the learned parameters and the ground truth parameters
false_loss_delta_d = Lambda(lambda x: K.mean(K.square(x), axis=1))(delta_d)
print("delta_d loss shape: " + str(false_loss_delta_d.shape))
#exit(1)
false_loss_delta_d = Reshape(target_shape=(1,), name="delta_d_mse")(false_loss_delta_d)
print("delta_d loss shape: " + str(false_loss_delta_d.shape))
# Load SMPL model and get necessary parameters
optlearner_pc = get_pc(optlearner_params, smpl_params, input_info, faces) # UNCOMMENT
print("optlearner_pc shape: " + str(optlearner_pc.shape))
#exit(1)
#optlearner_pc = Dense(6890*3)(delta_d)
#optlearner_pc = Reshape((6890, 3))(optlearner_pc)
# Get the (batched) Euclidean loss between the learned and ground truth point clouds
pc_euclidean_diff = Lambda(lambda x: x[0] - x[1])([gt_pc, optlearner_pc])
pc_euclidean_dist = Lambda(lambda x: K.sum(K.square(x),axis=-1))(pc_euclidean_diff)
print('pc euclidean dist '+str(pc_euclidean_dist.shape))
#exit(1)
false_loss_pc = Lambda(lambda x: K.mean(x, axis=1))(pc_euclidean_dist)
false_loss_pc = Reshape(target_shape=(1,), name="pc_mean_euc_dist")(false_loss_pc)
print("point cloud loss shape: " + str(false_loss_pc.shape))
#exit(1)
# Gather sets of points and compute their cross product to get mesh normals
# In order of: right hand, right wrist, right forearm, right bicep end, right bicep, right shoulder, top of cranium, left shoulder, left bicep, left bicep end, left forearm, left wrist, left hand,
# chest, belly/belly button, back of neck, upper back, central back, lower back/tailbone,
# left foot, left over-ankle, left shin, left over-knee, left quadricep, left hip, right, hip, right, quadricep, right over-knee, right shin, right, over-ankle, right foot
vertex_list = [5674, 5705, 5039, 5151, 4977, 4198, 411, 606, 1506, 1682, 1571, 2244, 2212,
3074, 3500, 460, 2878, 3014, 3021,
3365, 4606, 4588, 4671, 6877, 1799, 5262, 3479, 1187, 1102, 1120, 6740]
#face_array = np.array([11396, 8620, 7866, 5431, 6460, 1732, 4507])
pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
vertex_diff_NOGRAD = Lambda(lambda x: K.tf.gather(x, np.array(vertex_list).astype(np.int32), axis=-2))(pc_euclidean_diff_NOGRAD)
print("vertex_diff_NOGRAD shape: " + str(vertex_diff_NOGRAD.shape))
vertex_diff_NOGRAD = Flatten()(vertex_diff_NOGRAD)
#exit(1)
face_array = np.array([[face for face in faces if vertex in face][0] for vertex in vertex_list]) # only take a single face for each vertex
print("face_array shape: " + str(face_array.shape))
gt_normals = get_mesh_normals(gt_pc, face_array, layer_name="gt_cross_product")
print("gt_normals shape: " + str(gt_normals.shape))
opt_normals = get_mesh_normals(optlearner_pc, face_array, layer_name="opt_cross_product")
print("opt_normals shape: " + str(opt_normals.shape))
#exit(1)
# Learn the offset in parameters from the difference between the ground truth and learned mesh normals
diff_normals = Lambda(lambda x: K.tf.cross(x[0], x[1]), name="diff_cross_product")([gt_normals, opt_normals])
diff_normals_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_normals) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
diff_angles = Lambda(lambda x: K.tf.subtract(x[0], x[1]), name="diff_angle")([gt_normals, opt_normals])
diff_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_angles)
diff_angles_norm_NOGRAD = Lambda(lambda x: K.tf.norm(x, axis=-1), name="diff_angle_norm")(diff_angles_NOGRAD)
dist_angles = Lambda(lambda x: K.mean(K.square(x), axis=-1), name="diff_angle_mse")(diff_angles)
dist_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(dist_angles)
print("diff_angles shape: " + str(diff_angles.shape))
print("dist_angles shape: " + str(dist_angles.shape))
#pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
#print("diff_normals_NOGRAD shape: " + str(diff_normals_NOGRAD.shape))
diff_normals_NOGRAD = Flatten()(diff_normals_NOGRAD)
diff_angles_NOGRAD = Flatten()(diff_angles_NOGRAD)
mesh_diff_NOGRAD = Concatenate()([diff_normals_NOGRAD, dist_angles_NOGRAD])
if input_type == "3D_POINTS":
#optlearner_architecture = Dense(2**9, activation="relu")(vertex_diff_NOGRAD)
optlearner_architecture = Dense(2**7, activation="relu")(vertex_diff_NOGRAD)
if input_type == "MESH_NORMALS":
#optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_norm_NOGRAD)
#optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_NOGRAD)
#optlearner_architecture = Dense(2**9, activation="relu")(mesh_diff_NOGRAD)
optlearner_architecture = Dense(2**7, activation="relu")(mesh_diff_NOGRAD)
#optlearner_architecture = BatchNormalization()(optlearner_architecture)
#optlearner_architecture = Dropout(0.5)(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = Reshape((optlearner_architecture.shape[1].value, 1))(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = Conv1D(64, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = MaxPooling1D(3)(optlearner_architecture)
optlearner_architecture = Conv1D(128, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = MaxPooling1D(2)(optlearner_architecture)
optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = MaxPooling1D(2)(optlearner_architecture)
optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
#optlearner_architecture = MaxPooling1D(2)(optlearner_architecture)
optlearner_architecture = AveragePooling1D(2)(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = Flatten()(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#optlearner_architecture = Dropout(0.5)(optlearner_architecture)
optlearner_architecture = Dense(2**7, activation="relu")(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#delta_d_hat = Dense(85, activation=pos_scaled_tanh, name="delta_d_hat")(optlearner_architecture)
delta_d_hat = Dense(85, activation="linear", name="delta_d_hat")(optlearner_architecture)
#delta_d_hat = Dense(85, activation=centred_linear, name="delta_d_hat")(optlearner_architecture)
print('delta_d_hat shape: '+str(delta_d_hat.shape))
#exit(1)
# Calculate the (batched) MSE between the learned and ground truth offset in the parameters
delta_d_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(delta_d)
false_loss_delta_d_hat = Lambda(lambda x: K.mean(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat])
#false_loss_delta_d_hat = Lambda(lambda x: K.sum(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat])
#false_loss_delta_d_hat = Lambda(lambda x: mape(x[0], x[1]))([delta_d_NOGRAD, delta_d_hat])
false_loss_delta_d_hat = Reshape(target_shape=(1,), name="delta_d_hat_mse")(false_loss_delta_d_hat)
print("delta_d_hat loss shape: " + str(false_loss_delta_d_hat.shape))
#false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat)
false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat, average=False)
false_sin_loss_delta_d_hat = Lambda(lambda x: x, name="delta_d_hat_sin_output")(false_sin_loss_delta_d_hat)
print("delta_d_hat sin loss shape: " + str(false_sin_loss_delta_d_hat.shape))
# Prevent model from using the delta_d_hat gradient in final loss
delta_d_hat_NOGRAD = Lambda(lambda x: K.stop_gradient(x), name='optlearner_output_NOGRAD')(delta_d_hat)
# False loss designed to pass the learned offset as a gradient to the embedding layer
false_loss_smpl = Multiply(name="smpl_diff")([optlearner_params, delta_d_hat_NOGRAD])
print("smpl loss shape: " + str(false_loss_smpl.shape))
#return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, delta_d_hat_NOGRAD]
return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, dist_angles]
def build(self, input_shape):
param_shape_unshared = list(input_shape[1:])
param_shape_shared = list(input_shape[1:])
param_broadcast_unshared = [False] * len(param_shape_unshared)
param_broadcast_shared = [False] * len(param_shape_shared)
if self.shared_axes is not None:
for i in self.shared_axes:
param_shape_shared[i - 1] = 1
param_broadcast_shared[i - 1] = True
self.a_param_broadcast = param_broadcast_shared
self.k_param_broadcast = param_broadcast_shared
self.n_param_broadcast = param_broadcast_shared
self.z_param_broadcast = param_broadcast_shared
a_param_shape = param_shape_shared
k_param_shape = param_shape_shared
n_param_shape = param_shape_shared
z_param_shape = param_shape_shared
if not self.a_shared:
a_param_shape = param_shape_unshared
self.a_param_broadcast = param_broadcast_unshared
if not self.k_shared:
k_param_shape = param_shape_unshared
self.k_param_broadcast = param_broadcast_unshared
if not self.n_shared:
n_param_shape = param_shape_unshared
self.n_param_broadcast = param_broadcast_unshared
if not self.z_shared:
z_param_shape = param_shape_unshared
self.z_param_broadcast = param_broadcast_unshared
self.a = self.add_weight(shape=a_param_shape,
name='a',
initializer=self.a_initializer,
regularizer=self.a_regularizer,
constraint=self.a_constraint)
self.k = self.add_weight(shape=k_param_shape,
name='k',
initializer=self.k_initializer,
regularizer=self.k_regularizer,
constraint=self.k_constraint)
self.n = self.add_weight(shape=n_param_shape,
name='n',
initializer=self.n_initializer,
regularizer=self.n_regularizer,
constraint=self.n_constraint)
if not self.z_one:
self.z = self.add_weight(shape=z_param_shape,
name='z',
initializer=self.z_initializer,
regularizer=self.z_regularizer,
constraint=self.z_constraint)
else:
self.z = K.constant(1.)
# Set input spec
axes = {}
if self.shared_axes:
for i in range(1, len(input_shape)):
if i not in self.shared_axes:
axes[i] = input_shape[i]
self.input_spec = InputSpec(ndim=len(input_shape), axes=axes)
self.built = True
dataset[np.arange(n),
np.random.choice(range(len(probs)), p=probs, size=n, replace=True)] = 1
# model = Sequential()
# model.add(Dense(units=len(probs), activation='softmax', input_dim=1, activity_regularizer=IndepedentRegularizer((2,2,2), weight=1.)))
# model.compile(optimizer='sgd', loss='categorical_crossentropy')
# model.fit(np.zeros((n, 1)), dataset, batch_size=10, verbose=False)
# print(entropy(probs, model.predict(np.array([[0]]))[0]))
# model = Sequential()
# model.add(Dense(units=len(probs), activation='softmax', input_dim=1))
# model.compile(optimizer='sgd', loss='categorical_crossentropy')
# model.fit(np.zeros((n, 1)), dataset, batch_size=10, verbose=False)
# print(entropy(probs, model.predict(np.array([[0]]))[0]))
reg = IndepedentRegularizer((2, 2, 2, 2, 2, 2), 10.)
freqs = K.constant((1 + np.sum(dataset, axis=0)) / (n + len(probs)))
prob_tensor = K.variable(np.sum(dataset, axis=0) / n)
prob_tensor = prob_tensor / tf.reduce_sum(prob_tensor)
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(
reg(prob_tensor) + kullback_leibler_divergence(freqs, prob_tensor))
init = tf.initialize_all_variables()
with tf.Session() as session:
session.run(init)
for step in range(1000):
session.run(train)
print("step", step, "entropy:", entropy(probs,
session.run(prob_tensor)))
print(session.run(prob_tensor))
self.loss_value = None
self.grad_values = None
return grad_values
# This is the path to the image you want to transform.
TARGET_IMG = 'tiger.jpeg'
# This is the path to the style image.
REFERENCE_STYLE_IMG = 'style3.png'
width, height = load_img(TARGET_IMG).size
img_height = 320
img_width = int(width * img_height / height)
target_image = K.constant(preprocess_image(TARGET_IMG, height=img_height, width=img_width))
style_image = K.constant(preprocess_image(REFERENCE_STYLE_IMG, height=img_height, width=img_width))
# Placeholder for our generated image
generated_image = K.placeholder((1, img_height, img_width, 3))
# Combine the 3 images into a single batch
input_tensor = K.concatenate([target_image,
style_image,
generated_image], axis=0)
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
layers = dict([(layer.name, layer.output) for layer in model.layers])
def VSR(scale=4, depth=7, num_blocks=8, nD=4, name=None):
if depth % 2 == 0:
raise ValueError("bad depth")
Input_Tensor = Input((depth, None, None, 3))
X_in = Normalization()(Input_Tensor)
X = ZeroPadding3D(padding=(0, 3, 3))(X_in)
X = Conv3D(filters=64,
kernel_size=(1, 7, 7),
strides=(1, 1, 1),
padding='valid')(X)
# residual dense blocks
for i in range(num_blocks):
X = ResidualDenseBlock(X, 64, 64, 16, nD)
pass
# depth turn to 1
num_conv = int((depth - 1) / 2)
nF = 128
nG = 64
for i in range(num_conv):
T = BatchNormalization(axis=4)(X)
T = LeakyReLU(alpha=0.1)(T)
T = Conv3D(filters=nF,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid')(T)
T = BatchNormalization(axis=4)(T)
T = LeakyReLU(alpha=0.1)(T)
T = ZeroPadding3D(padding=(0, 1, 1))(T)
T = Conv3D(filters=nG,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='valid')(T)
S = Lambda(lambda x: x[:, 1:-1])(X)
X = concatenate([S, T], axis=4)
nF += nG
pass
X = BatchNormalization(axis=4)(X)
X = LeakyReLU(alpha=0.1)(X)
X = ZeroPadding3D(padding=(0, 1, 1))(X)
X = Conv3D(filters=256,
kernel_size=(1, 3, 3),
strides=(1, 1, 1),
padding='valid')(X)
X = LeakyReLU(alpha=0.1)(X)
R = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid')(X)
R = LeakyReLU(alpha=0.1)(R)
R = Conv3D(filters=3 * scale * scale,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid')(R)
F = Conv3D(filters=512,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid')(X)
F = LeakyReLU(alpha=0.1)(F)
F = Conv3D(filters=1 * 5 * 5 * scale * scale,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid')(F)
shape = (K.shape(F)[0], K.shape(F)[1], K.shape(F)[2], K.shape(F)[3], 25,
scale * scale)
F = Lambda(K.reshape, arguments={'shape': shape})(F)
F = Softmax(axis=4)(F)
# dynamic filter
filter_localexpand_np = np.reshape(np.eye(25, 25), (5, 5, 1, 25))
filter_localexpand = K.constant(filter_localexpand_np, dtype='float32')
x_channals = []
for c in range(3):
x = Lambda(lambda x: x[:, depth // 2:depth // 2 + 1, :, :, c])(X_in)
f = Lambda(lambda x: x[:, 0, :, :, :, :])(F)
x = Lambda(tf.transpose, arguments={'perm': [0, 2, 3, 1]})(x)
x_localexpand = Lambda(K.conv2d, arguments={'kernel':filter_localexpand, \
'strides':(1, 1), 'padding':'same'})(x)
x_localexpand = Lambda(tf.expand_dims, arguments={'axis':
3})(x_localexpand)
x = Lambda(tf.matmul, arguments={'b': f})(x_localexpand)
x = Lambda(tf.squeeze, arguments={'axis': 3})(x)
x = SubpixelConv2D(scale=scale)(x)
x_channals.append(x)
pass
x = concatenate(x_channals, axis=3)
x = Lambda(tf.expand_dims, arguments={'axis': 1})(x)
# depth to space 3D
r_shape = K.shape(R)
shape = (r_shape[0] * r_shape[1], r_shape[2], r_shape[3], r_shape[4])
r = Lambda(K.reshape, arguments={'shape': shape})(R)
y = Lambda(tf.depth_to_space, arguments={'block_size': scale})(r)
y_shape = K.shape(y)
shape = (r_shape[0], r_shape[1], y_shape[1], y_shape[2], y_shape[3])
r = Lambda(K.reshape, arguments={'shape': shape})(y)
# add
G = layers.add([x, r])
G = Lambda(lambda x: x[:, 0, :, :, :])(G)
Output_Tensor = Denormalization()(G)
if name == None:
name = "VSR-" + str(num_blocks) + "-" + str(nD) + "-x" + str(scale)
pass
model = Model(Input_Tensor, Output_Tensor, name=name)
return model
def loss_func_ck_default():
return K.constant(0.0)
def __init__(self, select, **kwargs):
super(Selector, self).__init__(**kwargs)
self.select = select
self.select_neuron = K.constant(value=self.select)
def one_elem_loss_generator(id_elem, elem_param, basis_info, model_info):
"""
build loss function for one element
:param id_elem: id of current element and sub-model
:param elem_param: tuple, (n_modes, n_quads, lambda_coeff, bc, jacobian, domain, ck_k)
where n_modes is the number of modes in each element
n_quads is the number of quadrature points in each element
lambda_coeff is the lambda coefficient
bc : numpy array, shape: (2,2); bc[:, 0] contains coordinates of 2 boundaries
bc[:, 1] contains Dirivhlet BC values of 2 boundaries
jacobian: vector containing jacobians of each element, shape: (n_elem,)
domain: vector contains element boundary coordinates, shape: (n_elem+1,)
ck_k : k value in C_k continuity
:param basis_info: tuple, (B, Bd, W, B_bound)
where B : basis matrix, shape: (n_quads, n_modes)
Bd : basis derivative matrix, shape: (n_quads, n_modes)
W : weight matrix, shape: (1, n_quads)
B_bound : matrix of basis values on x=-1, 1; shape: (2, n_modes)
:param model_info: tuple, (model, sub_model_set)
where model: multi_element_model for n_elem elements, overall model of n_elem sub-models
sub_model_set: list of sub-models for each element, shape: (n_elem,)
:return: loss function for the sub-model of this element
if this is the first element, will compute equation residual of this element, and also of element-boundary
C^k continuity conditions, as well as global boundary condition loss
if this is not the first element, will compute only the equation residual of this element
"""
n_modes, n_quads, lambda_coeff, bc, jacob, this_domain, ck_k = elem_param
n_elem = jacob.shape[0] # number of elements
global_model, sub_model_list = model_info # global_model contains the overall model
this_sub_model = sub_model_list[id_elem] # sub_model for this element
first_sub_model = sub_model_list[0] # sub_model for first element
last_sub_model = sub_model_list[n_elem - 1] # sub_model for last element
B_trans, Bd_trans, W_mat_A, B_bound = basis_info
# B_trans contains basis function matrix, shape: (n_quads, n_modes)
# Bd_trans contains basis derivative matrix, shape: (n_quads, n_modes)
# W_mat_A contains weight matrix, shape: (1, n_quads)
# B_bound contains boundary basis matrix, shape: (2, n_modes)
W_mat = np.zeros((n_elem, n_quads))
W_A = np.zeros((n_elem, n_quads))
for i in range(n_elem):
W_mat[i, :] = W_mat_A[0, :] * jacob[i]
W_A[i, :] = W_mat_A[0, :]
# now W_mat contains jacobian * W matrix, shape: (n_elem, n_quads)
# W_A contains W matrix, shape: (n_elem, n_quads)
B_bound_trans = np.copy(B_bound) # shape: (2, n_modes)
B_bound_trans[0, :] = -B_bound_trans[0, :]
# negate value at x=-1, because in weak form we have -phi(-1)*du/dx|_{x=-1}
B_bound_left = np.zeros((n_elem, n_modes, 1)) # shape: (n_elem,n_quads,1)
B_bound_right = np.zeros((n_elem, n_modes, 1)) # shape: (n_elem,n_quads,1)
B_bound_left[0, :, 0] = B_bound_trans[
0, :] # modes of first element, left boundary x=a
B_bound_right[-1, :, 0] = B_bound_trans[
1, :] # modes of right element, right boundary x=b
## ============================================##
# generate Keras tensors from these matrices or vectors
B_tensor = K.constant(
B_trans) # in standard element, shape: (n_quads, n_modes)
W_mat_tensor = K.constant(
W_mat) # shape of W_mat_tensor: (n_elem, n_quads)
W_mat_A_tensor = K.constant(W_A) # shape : (n_elem, n_quads)
Bd_tensor = K.constant(
Bd_trans) # in standard element, shape; (n_quads, n_modes)
# B_bound_tensor = K.constant(B_bound_trans) # shape: (2, n_modes)
B_bound_left_tensor = K.constant(
B_bound_left) # shape: (n_elem, n_modes, 1)
B_bound_right_tensor = K.constant(
B_bound_right) # shape: (n_elem, n_modes, 1)
# ======================================= #
# element boundary du/dx
# Zb = np.zeros((2, 1))
# Zb[0, 0] = this_domain[id_elem]
# Zb[1, 0] = this_domain[id_elem+1]
# now Zb_elem contains coordinates of end points for this element
# Zb_tensor = K.constant(Zb)
# Zb_input_tensor = keras.Input(tensor = Zb_tensor)
# now Zb_input_tensor contains input tensor for this sub-model
# ======================================= #
# boundary conditions for domain
ZL = np.zeros((1, 1))
ZL[0, 0] = bc[0, 0] # left boundary coordinate
ZL_tensor = K.constant(ZL)
# convert ZL_tensor into Keras input tensor
ZL_input_tensor = keras.Input(tensor=ZL_tensor)
ZR = np.zeros((1, 1))
ZR[0, 0] = bc[1, 0] # right boundary coordinate
ZR_tensor = K.constant(ZR)
ZR_input_tensor = keras.Input(tensor=ZR_tensor)
BCL = np.zeros((1, 1))
BCL[0, 0] = bc[0, 1] # left boundary condition
BCL_tensor = K.constant(BCL) # shape: (1,1)
BCR = np.zeros((1, 1))
BCR[0, 0] = bc[1, 1] # right boundary condition
BCR_tensor = K.constant(BCR) # shape: (1,1)
# ====================================== #
# continuity across element boundaries
Zb_elem = np.zeros((n_elem, 2, 1))
for i in range(n_elem):
Zb_elem[i, 0, 0] = this_domain[i]
Zb_elem[i, 1, 0] = this_domain[i + 1]
Zb_elem_tensor = []
for i in range(n_elem):
temp_tensor = K.constant(Zb_elem[i]) # shape (2,1)
temp_input_tensor = keras.Input(tensor=temp_tensor)
Zb_elem_tensor.append(temp_input_tensor)
# now Zb_elem_tensor contains list of element-boundary coordinate tensors
# element-boundary value difference
bound_assembly = np.zeros((n_elem - 1, n_elem * 2))
for i in range(n_elem - 1):
bound_assembly[
i, 2 * i +
1] = 1.0 # right boundary of i-th element, the one on the left
bound_assembly[i, 2 * (
i + 1
)] = -1.0 # left boundary of (i+1)-th element, the one on the right
bound_assembly_tensor = K.constant(
bound_assembly) # shape: (n_elem-1, n_elem*2)
# to be used for computing different between values across element boundaries
# ============================= #
# element boundary mode assembly matrix and tensor
bmode_assembler = np.zeros((n_elem + 1, 2 * n_elem))
bmode_assembler[0, 0] = 1.0
bmode_assembler[-1, -1] = 1.0
for i in range(n_elem - 1):
bmode_assembler[i + 1, 2 * i + 1] = 1.0 # i-th element, right boundary
bmode_assembler[i + 1,
2 * (i + 1)] = 1.0 # (i+1)-th element, left boundary
# now bmode_assembler contains assembler matrix
bmode_assembler_tensor = K.constant(
bmode_assembler) # Keras tensor, shape: (n_elem+1, 2*n_elem)
# ============================= #
def loss_generator_Ck(ck):
"""
Ck loss function generator
:param ck: integer, >= 0, k value in C_k continuity
:return: loss function for C_k
"""
def loss_func_c0():
"""
compute loss across element boundary for C^0 continuity across element boundaries
:return: loss value associated with C0 continuity condition
"""
# now Zb_elem_tensor contains a list of (2,1) element boundary coordinate tensors
output_tensor = global_model(Zb_elem_tensor)
# now output_tensor contains list of output tensors, shape: (n_elem, 2, 1)
temp_out_tensor = K.concatenate(output_tensor,
0) # concatenate list tensors
# now temp_out_tensor has shape (n_elem*2, 1)
C0_residual_tensor = K.dot(bound_assembly_tensor, temp_out_tensor)
# C0_residual_tensor has shape: (n_elem-1, 1)
this_loss = K.sum(K.square(C0_residual_tensor))
return this_loss
def loss_func_c1():
"""
compute loss across element boundary for C^1 continuity across element boundaries
:return: loss value
"""
output_tensor = global_model(Zb_elem_tensor)
# now output_tensor contains list of output tensors, shape: (n_elem, 2, 1)
grad_dudx = K.gradients(output_tensor, Zb_elem_tensor)
# now grad_dudx contains gradients,
# shape is (n_elem, 2, 1)
# grad_dudx is a list of (2,1) arrays
C0_concat = K.concatenate(output_tensor, 0) # shape (n_elem*2, 1)
C1_concat = K.concatenate(grad_dudx, 0) # shape (n_elem*2, 1)
C0_residual = K.dot(bound_assembly_tensor,
C0_concat) # shape: (n_elem-1, 1)
C1_residual = K.dot(bound_assembly_tensor,
C1_concat) # shape: (n_elem-1, 1)
this_loss = K.sum(K.square(C0_residual)) + K.sum(
K.square(C1_residual))
return this_loss
def loss_func_c2():
"""
loss for C^2 continuity across element boundaries
:return: loss value
"""
ub = global_model(
Zb_elem_tensor
) # ub constains element boundary value tensor, shape: (n_elem, 2, 1)
dub_dx = K.gradients(
ub, Zb_elem_tensor
) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1)
dub_dx_2 = K.gradients(dub_dx, Zb_elem_tensor)
# dud_dx_2 contains d^2u/dx^2 tensor on boundary, shape: (n_elem,2,1)
# Note: tensorflow.gradients sums up all w.r.t. to list of output tensors
# but since only du1/dx1, du2/dx2, du3/dx3 ... is non-zero, while all du_i/dx_j=0 if i!=j
# dudx effectively contains only du_i/dx_i, for i=0, ..., n_elem-1
# similarly, for second derivatives, du_dx_2 contains only d^2 u_i/dx_i^2, for i=0,...,n_elem-1 only
C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1)
C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1)
C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1)
C0_residual = K.dot(bound_assembly_tensor,
C0_concat) # shape: (n_elem-1, 1)
C1_residual = K.dot(bound_assembly_tensor,
C1_concat) # shape: (n_elem-1, 1)
C2_residual = K.dot(bound_assembly_tensor,
C2_concat) # shape: (n_elem-1, 1)
this_loss = K.sum(K.square(C0_residual)) + K.sum(
K.square(C1_residual)) + K.sum(K.square(C2_residual))
return this_loss
def loss_func_c3():
"""
compute loss associated with C^3 continuity
:return: loss value
"""
ub = global_model(
Zb_elem_tensor
) # ub constains element boundary value tensor, shape: (n_elem, 2, 1)
dub_dx = K.gradients(
ub, Zb_elem_tensor
) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1)
dub_dx_2 = K.gradients(
dub_dx, Zb_elem_tensor
) # second derivative tensor, shape: (n_elem,2,1)
dub_dx_3 = K.gradients(
dub_dx_2,
Zb_elem_tensor) # third derivative tensor, shape: (n_elem,2,1)
C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1)
C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1)
C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1)
C3_concat = K.concatenate(dub_dx_3, 0) # shape: (n_elem*2,1)
C0_residual = K.dot(bound_assembly_tensor,
C0_concat) # shape: (n_elem-1, 1)
C1_residual = K.dot(bound_assembly_tensor,
C1_concat) # shape: (n_elem-1, 1)
C2_residual = K.dot(bound_assembly_tensor,
C2_concat) # shape: (n_elem-1, 1)
C3_residual = K.dot(bound_assembly_tensor,
C3_concat) # shape: (n_elem-1, 1)
this_loss = K.sum(K.square(C0_residual)) + K.sum(K.square(C1_residual)) \
+ K.sum(K.square(C2_residual)) + K.sum(K.square(C3_residual))
return this_loss
def loss_func_c4():
"""
compute loss associated with C^4 continuity
:return: loss value
"""
ub = global_model(
Zb_elem_tensor
) # ub constains element boundary value tensor, shape: (n_elem, 2, 1)
dub_dx = K.gradients(
ub, Zb_elem_tensor
) # dub_dx contains du/dx tensor on boundary, shape: (n_elem,2,1)
dub_dx_2 = K.gradients(
dub_dx, Zb_elem_tensor
) # second derivative tensor, shape: (n_elem,2,1)
dub_dx_3 = K.gradients(
dub_dx_2,
Zb_elem_tensor) # third derivative tensor, shape: (n_elem,2,1)
dub_dx_4 = K.gradients(
dub_dx_3, Zb_elem_tensor
) # fourth derivative tensor, shape: (n_elem,2,1)
C0_concat = K.concatenate(ub, 0) # shape: (n_elem*2, 1)
C1_concat = K.concatenate(dub_dx, 0) # shape: (n_elem*2, 1)
C2_concat = K.concatenate(dub_dx_2, 0) # shape: (n_elem*2, 1)
C3_concat = K.concatenate(dub_dx_3, 0) # shape: (n_elem*2,1)
C4_concat = K.concatenate(dub_dx_4, 0) # shape: (n_elem*2, 1)
C0_residual = K.dot(bound_assembly_tensor,
C0_concat) # shape: (n_elem-1, 1)
C1_residual = K.dot(bound_assembly_tensor,
C1_concat) # shape: (n_elem-1, 1)
C2_residual = K.dot(bound_assembly_tensor,
C2_concat) # shape: (n_elem-1, 1)
C3_residual = K.dot(bound_assembly_tensor,
C3_concat) # shape: (n_elem-1, 1)
C4_residual = K.dot(bound_assembly_tensor,
C4_concat) # shape: (n_elem-1, 1)
this_loss = K.sum(K.square(C0_residual)) + K.sum(K.square(C1_residual)) \
+ K.sum(K.square(C2_residual)) + K.sum(K.square(C3_residual)) \
+ K.sum(K.square(C4_residual))
return this_loss
def loss_func_ck_default():
return K.constant(0.0)
# ++++++++++++ #
if ck == 0:
return loss_func_c0
elif ck == 1:
return loss_func_c1
elif ck == 2:
return loss_func_c2
elif ck == 3:
return loss_func_c3
elif ck == 4:
return loss_func_c4
else:
print(
"ERROR: loss_generator_ck() -- C^k continuity with (infinity > k > 4) is not implemented!\n"
)
return loss_func_ck_default
# +++++++++++++ #
The_Loss_Func_Ck = loss_generator_Ck(ck_k)
# now The_Loss_Func_Ck is the loss function for computing Ck continuity loss
# across element boundaries
# ======================================= #
def Equation_Residual(left_tensor_pair, right_tensor_pair):
"""
actual computation of residual tensor for equation for all elements
:param: left_tensor_pair: tuple, (BCL_in_tensor, BCL_out_tensor)
where BCL_in_tensor is the input tensor for left domain boundary, shape: (1,1)
BCL_out_tensor is the output tensor for left domain boundary, shape: (1,1)
:param: right_tensor_pair: tuple, (BCR_in_tensor, BCR_out_tensor)
where BCR_in_tensor is the input tensor for right domain boundary, shape: (1,1)
BCR_out_tensor is the output tensor for right domain boundary, shape: (1,1)
:return: equation residual tensor for all elements
Note: when using tensorflow.gradients to compute derivative of a list of output tensors (u_i) with respect to
a list of input tensors (x_j), the resultant gradient is a list with \sum_{i=0}^{N-1} du_i/dx_j, for
j = 0, 1, ..., M-1. Note that it is summed over i. So the length of resultant list is equal to
the length of the list of input tensors.
In this code, since different sub-models are not connected, i.e. du_i/dx_j = 0 if i != j, the resultant
sum will have no effect. So the resultant gradient is effectively du_i/dx_i, i=0, ..., M-1.
"""
# now global_model contains entire model
# sub_model_list contains the list of sub-models
# first_sub_model contains the DNN for the first element
# last_sub_model contains the DNN for the last element
# \sum du/dx * d_phi/dx*jacobian * w = \sum du/dx * d_phi/dxi * w
# where x is physical coordinate, xi is coordinate in standard element
In_tensors = global_model.inputs # list of input tensors, shape: (n_elem, n_quads, 1)
Out_tensors = global_model.outputs # list of output tensors, shape: (n_elem, n_quads, 1)
dudx = K.gradients(
Out_tensors,
In_tensors) # list of du/dx tensors, shape: (n_elem, n_quads, 1)
dudx_con = K.concatenate(dudx, 0) # shape: (n_elem*n_quads, 1)
dudx_1 = K.reshape(dudx_con,
(n_elem, n_quads)) # shape: (n_elem, n_quads)
dudx_w = dudx_1 * W_mat_A_tensor # element wise multiply, shape: (n_elem, n_quads)
T1 = K.dot(dudx_w, Bd_tensor) # shape: (n_elem, n_modes)
# Note that Bd_tensor contains dphi/dxi in standard element, shape: (n_quads,n_modes)
# dphi/dx in physical space is dphi/dxi / jacobian
# now T1[n_modes, n_elem] contains first term in weak form
# lambda_coeff* \sum u*basis*jacobian * w
# now Out_tensors contains u on all quadrature points of all elements, list of (n_quads,1) tensors
# first concatenate them into a single tensor
u_con = K.concatenate(Out_tensors, 0) # shape: (n_elem*n_quads, 1)
u_1 = K.reshape(u_con, (n_elem, n_quads)) # shape: (n_elem, n_quads)
temp_v1 = lambda_coeff * u_1 * W_mat_tensor # note: W_mat_tensor includes jacobian, shape: (n_elem,n_quads)
T2 = K.dot(temp_v1, B_tensor) # shape: (n_elem, n_modes)
# now T2 contains second term, shape: (n_elem, n_modes)
# source term \sum f(x) * basis * jacobian * w
# Note: list of label tensors is contained in global_model.targets
target_tensors = global_model.targets # list of label tensors, shape: (n_elem, n_quads, ?)
label_tensors = K.concatenate(target_tensors,
0) # shape: (n_elem*n_quads, ?)
y_true_0 = label_tensors[:, 0] # shape: (n_elem*n_quads,)
y_true_1 = K.reshape(y_true_0,
(n_elem, n_quads)) # shape: (n_elem, n_quads)
temp_v2 = y_true_1 * W_mat_tensor # temp_v2 has shape (n_elem, n_quads)
T3 = K.dot(temp_v2, B_tensor) # shape: (n_elem, n_modes)
# now T3 contains source term, shape (n_elem, n_modes)
# boundary terms
left_in_tensor, left_out_tensor = left_tensor_pair
right_in_tensor, right_out_tensor = right_tensor_pair
dudx_a = K.gradients(
left_out_tensor,
left_in_tensor)[0] # gradient on left boundary, shape: (1,1)
dudx_b = K.gradients(
right_out_tensor,
right_in_tensor)[0] # gradient on right boundary, shape: (1,1)
# Note: returned data from tensorflow.gradients is a list of tensors, even though the length
# of the list may be 1. So we need the "...[0]" in the above to get the actual tensor
T_bound_left_1 = K.dot(B_bound_left_tensor,
dudx_a) # shape: (n_elem,n_modes,1)
T_bound_left = K.squeeze(T_bound_left_1,
-1) # shape: (n_elem, n_modes)
T_bound_right_1 = K.dot(B_bound_right_tensor,
dudx_b) # shape: (n_elem, n_modes, 1)
T_bound_right = K.squeeze(T_bound_right_1,
-1) # shape: (n_elem, n_modes)
T_bound_elem = T_bound_left + T_bound_right # shape: (n_elem, n_modes)
# now T_bound_elem contains contribution of boundary terms
# -dudx(a)*phi(a) + dudx(b)*phi(b)
# loss
T_tot = T1 + T2 + T3 - T_bound_elem # now T_tot has shape: (n_elem, n_modes)
return T_tot
def The_Loss_Func(y_true, y_pred):
"""
actual computation of loss function for ordinary element, not the first element
only compute the residual loss of equation for current element
:param y_true: label data
:param y_pred: preduction data
:return: loss value
"""
# DNN corresponding to other elements will simply return 0 for loss
# DNN corresponding to first element will compute the loss for all elements
this_loss = K.constant(0.0)
return this_loss
def The_First_Loss_Func(y_true, y_pred):
"""
actual computation of loss for all elements
compute residual loss of equation for all elements, and loss in continuity across element
boundary, and loss in boundary condition of domain
:param y_true:
:param y_pred:
:return:
"""
# ========================== #
# domain boundary condition
BCL_out = first_sub_model(ZL_input_tensor) # shape: (1,1)
T_bc_L = BCL_out - BCL_tensor # shape: (1,1)
BCR_out = last_sub_model(ZR_input_tensor) # shape: (1,1)
T_bc_R = BCR_out - BCR_tensor # shape: (1,1)
# ========================= #
# equation residual
bcl_pair = (ZL_input_tensor, BCL_out)
bcr_pair = (ZR_input_tensor, BCR_out)
T_tot = Equation_Residual(bcl_pair, bcr_pair)
# now T_tot contains residual tensor for equation of all elements, shape: (n_elem, n_modes)
# ========================== #
# extract element boundary and interior modes
# assemble boundary modes
bmode = K.slice(T_tot, (0, 0), (n_elem, 2))
# bmode contains boundary mode contributions, shape: (n_elem, 2)
interior_mode = K.slice(T_tot, (0, 2), (n_elem, n_modes - 2))
# interior_mode contains interior mode contributions, shape: (n_elem, n_modes-2)
bmode_2 = K.reshape(bmode, (n_elem * 2, 1)) # shape (n_elem*2,1)
bmode_glob = K.dot(bmode_assembler_tensor, bmode_2)
# now bmode_glob contains global mode contributions, shape: (n_elem+1,1)
# interior_mode contains interior mode contributions
value_1 = K.flatten(bmode_glob)
value_2 = K.flatten(interior_mode)
value = K.concatenate((value_1, value_2))
# now value contains all global modes, shape: (n_elem+1+n_elem*(n_modes-2),), 1D array
# ========================= #
# C^k continuity across element boundary
Ck_loss = The_Loss_Func_Ck()
# now Ck_loss contains loss value for C_k continuity across element boundaries
# ========================= #
this_loss = K.mean(K.square(value)) \
+ (K.sum(K.square(T_bc_L)) + K.sum(K.square(T_bc_R)) + Ck_loss)
return this_loss
# ====================================== #
if id_elem == 0:
return The_First_Loss_Func
else:
return The_Loss_Func
def attention(self,
pre_q,
pre_v,
pre_k,
out_seq_len: int,
d_model: int,
training=None,
padding_mask=None):
"""
Calculates the output of the attention once the affine transformations
of the inputs are done. Here's the shapes of the arguments:
:param pre_q: (batch_size, q_seq_len, num_heads, d_model // num_heads)
:param pre_v: (batch_size, v_seq_len, num_heads, d_model // num_heads)
:param pre_k: (batch_size, k_seq_len, num_heads, d_model // num_heads)
:param out_seq_len: the length of the output sequence
:param d_model: dimensionality of the model (by the paper)
:param training: Passed by Keras. Should not be defined manually.
Optional scalar tensor indicating if we're in training
or inference phase.
"""
# shaping Q and V into (batch_size, num_heads, seq_len, d_model//heads)
q = K.permute_dimensions(pre_q, [0, 2, 1, 3])
v = K.permute_dimensions(pre_v, [0, 2, 1, 3])
if self.compression_window_size is None:
k_transposed = K.permute_dimensions(pre_k, [0, 2, 3, 1])
else:
# Memory-compressed attention described in paper
# "Generating Wikipedia by Summarizing Long Sequences"
# (https://arxiv.org/pdf/1801.10198.pdf)
# It compresses keys and values using 1D-convolution which reduces
# the size of Q * K_transposed from roughly seq_len^2
# to convoluted_seq_len^2. If we use strided convolution with
# window size = 3 and stride = 3, memory requirements of such
# memory-compressed attention will be 9 times smaller than
# that of the original version.
if self.use_masking:
raise NotImplementedError(
"Masked memory-compressed attention has not "
"been implemented yet")
k = K.permute_dimensions(pre_k, [0, 2, 1, 3])
k, v = [
K.reshape(
# Step 3: Return the result to its original dimensions
# (batch_size, num_heads, seq_len, d_model//heads)
K.bias_add(
# Step 3: ... and add bias
K.conv1d(
# Step 2: we "compress" K and V using strided conv
K.reshape(
# Step 1: we reshape K and V to
# (batch + num_heads, seq_len, d_model//heads)
item,
(-1, K.int_shape(item)[-2],
d_model // self.num_heads)),
kernel,
strides=self.compression_window_size,
padding='valid',
data_format='channels_last'),
bias,
data_format='channels_last'),
# new shape
K.concatenate([
K.shape(item)[:2],
[
K.int_shape(item)[2] //
self.compression_window_size,
d_model // self.num_heads
]
])) for item, kernel, bias in ((k, self.k_conv_kernel,
self.k_conv_bias),
(v, self.v_conv_kernel,
self.v_conv_bias))
]
k_transposed = K.permute_dimensions(k, [0, 1, 3, 2])
# shaping K into (batch_size, num_heads, d_model//heads, seq_len)
# for further matrix multiplication
sqrt_d = K.constant(np.sqrt(d_model // self.num_heads),
dtype=K.floatx())
q_shape = K.int_shape(q)
k_t_shape = K.int_shape(k_transposed)
v_shape = K.int_shape(v)
# before performing batch_dot all tensors are being converted to 3D
# shape (batch_size * num_heads, rows, cols) to make sure batch_dot
# performs identically on all backends
attention_heads = K.reshape(
K.batch_dot(
self.apply_dropout_if_needed(K.softmax(
self.mask_attention_if_needed(
K.batch_dot(
K.reshape(q, (-1, ) + q_shape[-2:]),
K.reshape(k_transposed,
(-1, ) + k_t_shape[-2:])) / sqrt_d,
padding_mask)),
training=training),
K.reshape(v, (-1, ) + v_shape[-2:])),
(-1, self.num_heads, q_shape[-2], v_shape[-1]))
attention_heads_merged = K.reshape(
K.permute_dimensions(attention_heads, [0, 2, 1, 3]), (-1, d_model))
attention_out = K.reshape(
K.dot(attention_heads_merged, self.output_weights),
(-1, out_seq_len, d_model))
return attention_out
def style_preview(content, style, output):
width, height = load_img(content).size
img_height = height
img_width = width
target_image = K.constant(
preprocess_image(content, height=img_height, width=img_width))
style_image = K.constant(
preprocess_image(style, height=img_height, width=img_width))
# Placeholder for our generated image
generated_image = K.placeholder((1, img_height, img_width, 3))
# Combine the 3 images into a single batch
input_tensor = K.concatenate([target_image, style_image, generated_image],
axis=0)
stderr.write('Start download the model')
# The preview network needs to download to the suitable location
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False,
cache_dir=".")
stderr.write('Model downloading ending')
layers = dict([(layer.name, layer.output) for layer in model.layers])
# Set the content and style feature layers
content_layer, style_layers = set_cnn_layers(source=source_paper)
# initialize total loss
loss = K.variable(0.)
# add content loss
layer_features = layers[content_layer]
target_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss += content_weight * content_loss(target_image_features,
combination_features)
# add style loss
for layer_name in style_layers:
layer_features = layers[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
sl = style_loss(style_reference_features,
combination_features,
height=img_height,
width=img_width)
loss += (style_weight / len(style_layers)) * sl
# add total variation loss
loss += total_variation_weight * total_variation_loss(
generated_image, img_height, img_width)
# Get the gradients of the generated image wrt the loss
grads = K.gradients(loss, generated_image)[0]
# Function to fetch the values of the current loss and the current gradients
fetch_loss_and_grads = K.function([generated_image], [loss, grads])
evaluator = Evaluator(fetch_loss_and_grads=fetch_loss_and_grads,
height=img_height,
width=img_width)
# Run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the neural style loss.
# This is our initial state: the target image.
# Note that `scipy.optimize.fmin_l_bfgs_b` can only process flat vectors.
x = preprocess_image(content, height=img_height, width=img_width)
x = x.flatten()
for _ in range(iterations):
x, _, _ = fmin_l_bfgs_b(evaluator.loss,
x,
fprime=evaluator.grads,
maxfun=20)
img = x.copy().reshape((img_height, img_width, 3))
img = deprocess_image(img)
Image.fromarray(img).save(output, quality=95)
def __init__(self, bias):
super(ReverseBiasLayer, self).__init__()
self.bias = K.constant((-1) * bias)
