def yolo_eval(yolo_outputs,
anchors,
num_classes,
image_shape,
max_boxes=20,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input and return filtered boxes."""
num_layers = len(yolo_outputs)
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[
3, 4, 5
], [1, 2, 3]] # default setting
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
# with tf.Session() as sess:
# print(sess.run(input_shape[1]))
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats,
[-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
if calc_loss == True:
return grid, feats, box_xy, box_wh
return box_xy, box_wh, box_confidence, box_class_probs
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation('reshape', (4, 2), shape=(8, 1))
check_single_tensor_operation('permute_dimensions', (4, 2, 3),
pattern=(2, 0, 1))
check_single_tensor_operation('repeat', (4, 1), n=3)
check_single_tensor_operation('flatten', (4, 1))
check_single_tensor_operation('batch_flatten', (20, 2, 5))
check_single_tensor_operation('expand_dims', (4, 3), axis=-1)
check_single_tensor_operation('expand_dims', (4, 3, 2), axis=1)
check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
check_single_tensor_operation('squeeze', (4, 1, 1), axis=1)
check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)},
'squeeze', {'axis': 2}, (4, 3, 1, 1))
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation('reshape', (4, 2), shape=(8, 1))
check_single_tensor_operation('permute_dimensions', (4, 2, 3),
pattern=(2, 0, 1))
check_single_tensor_operation('repeat', (4, 1), n=3)
check_single_tensor_operation('flatten', (4, 1))
check_single_tensor_operation('expand_dims', (4, 3), dim=-1)
check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1)
check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
check_single_tensor_operation('squeeze', (4, 1, 1), axis=1)
check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)},
'squeeze', {'axis': 2},
(4, 3, 1, 1))
def build_discriminator(self):
model = Sequential()
## Head
if self.add_invmass_disc:
model.add(
Lambda(
add_invmass_from_8cartesian,
input_shape=self.output_shape,
output_shape=(self.output_shape[0] + 1, ),
))
else:
model.add(Dense(128, input_shape=self.output_shape))
if self.do_batch_normalization_disc:
model.add(BatchNormalization())
# model.add(LeakyReLU(alpha=0.2))
if self.do_concatenate_disc:
model.add(Lambda(lambda x: K.concatenate([x * x, x])))
model.add(LeakyReLU(alpha=0.2))
## Main Body
if self.depth_disc > 0 and self.width_disc > 0:
for level in xrange(0, self.depth_disc):
model.add(Dense(
width_disc /
(2**level))) #Triangle with width halved at each level
model.add(LeakyReLU(alpha=0.2))
else:
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(32))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(16))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(8))
model.add(LeakyReLU(alpha=0.2))
## Tail
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.output_shape)
validity = model(img)
return Model(img, validity)
def TSNs_SpatialStream(input_shape,
classes,
num_segments=3,
base_model='Xception',
dropout_prob=0.8,
consensus_type='avg',
partial_bn=True):
"""
Spatial stream of the Temporal Segment Networks (https://arxiv.org/pdf/1705.02953.pdf) defined as multi-input Keras model.
"""
# Define the shared layers, base conv net and enable partial batch
# normalization strategy
inputs = [Input(input_shape) for _ in range(num_segments)]
dropout = Dropout(dropout_prob)
dense = Dense(classes, activation=None)
act = Activation(activation='softmax', name='prediction')
models_dict = dict(getmembers(keras.applications, isfunction))
base = models_dict[base_model](include_top=False, pooling='avg')
if partial_bn:
num_bn_layers = 0
for layer in base.layers:
if isinstance(layer, BatchNormalization):
num_bn_layers += 1
if num_bn_layers != 1:
layer.trainable = False
# Pass multiple inputs (depending on num_segments) through the base conv
# net
outputs = []
visual_features = []
for seg_input in inputs:
seg_output = base(seg_input)
visual_features.append(seg_output)
seg_output = dropout(seg_output)
seg_output = dense(seg_output)
outputs.append(seg_output)
# Use a consensus function to combine class scores
if consensus_type == 'avg':
output = Average()(outputs)
elif consensus_type == 'max':
output = Maximum()(outputs)
elif consensus_type == 'attention':
weighted_outputs = []
attn_layer = Dense(1, use_bias=False, name='attn_layer')
attn_weights = [attn_layer(_) for _ in visual_features]
attn_weights = Lambda(lambda x: K.concatenate(x, axis=-1),
name='concatenate')(attn_weights)
attn_weights = Activation('softmax')(attn_weights)
for i, seg_output in enumerate(outputs):
weight = Lambda(lambda x: x[:, i])(attn_weights)
weighted_outputs.append(Multiply()([weight, seg_output]))
output = Add()(weighted_outputs)
output = act(output)
model = Model(inputs, output)
return model
def yolo_boxes_to_corners(graph, box_xy, box_wh):
with graph.as_default():
box_mins = box_xy - (box_wh / 2.)
box_maxes = box_xy + (box_wh / 2.)
return K.concatenate([
box_mins[..., 1:2], # y_min
box_mins[..., 0:1], # x_min
box_maxes[..., 1:2], # y_max
box_maxes[..., 0:1] # x_max
])
def fix_outputs(x):
"""
Take nominal delphes format of 19 columns and fix some columns
"""
return K.concatenate([
# x[:,0:21],
x[:, 0:7], # epxpypz for lep1,lep2 -1 for no py2
x[:, 7:8], # nvtx
K.sign(x[:, 8:10]), # q1 q2
x[:, 10:12], # iso1 iso2
x[:, 12:14], # met, metphi
x[:, 14:19], # jet pts
])
def yolo_correct_boxes(self, box_xy, box_wh, input_shape, image_shape):
'''Get corrected boxes'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_maxes = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], # y_min
box_mins[..., 1:2], # x_min
box_maxes[..., 0:1], # y_max
box_maxes[..., 1:2] # x_max
])
# Scale boxes back to original image shape.
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def fix_outputs(x):
"""
Take nominal delphes format of 21 columns and fix some columns
"""
return K.concatenate([
# x[:,0:21],
x[:, 0:8], # epxpypz for lep1,lep2
K.sign(x[:, 8:9]), # lep1 charge
x[:, 9:10], # lep1 iso
K.sign(x[:, 10:11]), # lep2 charge
x[:, 11:12], # lep2 iso
K.round(x[:, 12:13]), # nvtxs
x[:, 13:15], # met, metphi
K.round(x[:, 15:16]), # ngenjets
x[:, 16:21], # jet pts
])
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation("reshape", (4, 2), shape=(8, 1))
check_single_tensor_operation("permute_dimensions", (4, 2, 3), pattern=(2, 0, 1))
check_single_tensor_operation("repeat", (4, 1), n=3)
check_single_tensor_operation("flatten", (4, 1))
check_single_tensor_operation("expand_dims", (4, 3), dim=-1)
check_single_tensor_operation("expand_dims", (4, 3, 2), dim=1)
check_single_tensor_operation("squeeze", (4, 3, 1), axis=2)
def build_generator(self):
model = Sequential()
## Head
model.add(Dense(64, input_shape=self.noise_shape))
if self.do_batch_normalization_gen:
model.add(BatchNormalization())
model.add(LeakyReLU(alpha=0.2))
if self.do_concatenate_gen:
model.add(Lambda(lambda x: K.concatenate([x * x, x])))
model.add(LeakyReLU(alpha=0.2))
## Main Body
if self.depth_gen > 0 and self.width_gen > 0:
for level in xrange(0, self.depth_gen):
model.add(Dense(
width_gen /
(2**level))) #Triangle with width halved at each level
model.add(LeakyReLU(alpha=0.2))
elif self.beefy_generator:
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
else:
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(32))
model.add(LeakyReLU(alpha=0.2))
## Tail
model.add(Dense(self.output_shape[0]))
if self.do_tanh_gen:
model.add(Activation("tanh"))
elif self.fix_delphes_outputs:
model.add(
Lambda(fix_outputs,
input_shape=self.output_shape,
output_shape=self.output_shape))
model.summary()
noise = Input(shape=self.noise_shape)
img = model(noise)
return Model(noise, img)
def step(self, a, states):
r_tm1 = states[:self.nb_layers]
c_tm1 = states[self.nb_layers:2 * self.nb_layers]
e_tm1 = states[2 * self.nb_layers:3 * self.nb_layers]
if self.extrap_start_time is not None:
t = states[-1]
a = K.switch(
t >= self.t_extrap, states[-2], a
) # if past self.extrap_start_time, the previous prediction will be treated as the actual
c = []
r = []
e = []
# Update R units starting from the top
for l in reversed(range(self.nb_layers)):
inputs = [r_tm1[l], e_tm1[l]]
if l < self.nb_layers - 1:
inputs.append(r_up)
inputs = K.concatenate(inputs, axis=self.channel_axis)
i = self.conv_layers['i'][l].call(inputs)
f = self.conv_layers['f'][l].call(inputs)
o = self.conv_layers['o'][l].call(inputs)
_c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs)
_r = o * self.LSTM_activation(_c)
c.insert(0, _c)
r.insert(0, _r)
if l > 0:
r_up = self.upsample.call(_r)
# Update feedforward path starting from the bottom
for l in range(self.nb_layers):
ahat = self.conv_layers['ahat'][l].call(r[l])
if l == 0:
ahat = K.minimum(ahat, self.pixel_max)
frame_prediction = ahat
# compute errors
e_up = self.error_activation(ahat - a)
e_down = self.error_activation(a - ahat)
e.append(K.concatenate((e_up, e_down), axis=self.channel_axis))
if self.output_layer_num == l:
if self.output_layer_type == 'A':
output = a
elif self.output_layer_type == 'Ahat':
output = ahat
elif self.output_layer_type == 'R':
output = r[l]
elif self.output_layer_type == 'E':
output = e[l]
if l < self.nb_layers - 1:
a = self.conv_layers['a'][l].call(e[l])
a = self.pool.call(a) # target for next layer
if self.output_layer_type is None:
if self.output_mode == 'prediction':
output = frame_prediction
else:
for l in range(self.nb_layers):
layer_error = K.mean(K.batch_flatten(e[l]),
axis=-1,
keepdims=True)
all_error = layer_error if l == 0 else K.concatenate(
(all_error, layer_error), axis=-1)
if self.output_mode == 'error':
output = all_error
else:
output = K.concatenate(
(K.batch_flatten(frame_prediction), all_error),
axis=-1)
states = r + c + e
if self.extrap_start_time is not None:
states += [frame_prediction, t + 1]
return output, states
def add_invmass_from_8cartesian(x):
invmass = K.sqrt((x[:, 0:1] + x[:, 4:5])**2 - (x[:, 1:2] + x[:, 5:6])**2 -
(x[:, 2:3] + x[:, 6:7])**2 - (x[:, 3:4] + x[:, 7:8])**2)
return K.concatenate([x, invmass])
# this will contain our generated image
if K.image_dim_ordering() == 'th':
combination_image = K.placeholder((1, 3, img_width, img_height))
else:
combination_image = K.placeholder((1, img_width, img_height, 3))
image_tensor = [base_image]
for style_image_tensor in style_reference_images:
image_tensor.append(style_image_tensor)
image_tensor.append(combination_image)
nb_tensors = len(image_tensor)
nb_style_images = nb_tensors - 2 # Content andoutput image not considered
# combine the various image into a single Keras tensor
input_tensor = K.concatenate(image_tensor, axis=0)
if K.image_dim_ordering() == 'th':
shape = (nb_tensors, 3, img_width, img_height)
else:
shape = (nb_tensors, img_width, img_height, 3)
ip = Input(tensor=input_tensor, shape=shape)
# build a network
x = Conv2D(64, (3, 3), activation='relu', name='conv1_1', padding='same')(ip)
x = Conv2D(64, (3, 3), activation='relu', name='conv1_2', padding='same')(x)
x = pooling_func(x)
x = Conv2D(128, (3, 3), activation='relu', name='conv2_1', padding='same')(x)
x = Conv2D(128, (3, 3), activation='relu', name='conv2_2', padding='same')(x)
return tf.Session(config=tf.ConfigProto(
gpu_options=gpu_options, intra_op_parallelism_threads=num_threads))
else:
return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
import keras.backend.tensorflow_backend as KTF
KTF.set_session(get_session())
target_image = KTF.constant(preprocess_image(target_image_path))
style_reference_image = KTF.constant(
preprocess_image(style_reference_image_path))
combination_image = KTF.placeholder((1, img_height, img_width, 3))
# In[9]:
input_tensor = KTF.concatenate(
[target_image, style_reference_image, combination_image], axis=0)
# In[10]:
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
print('Model loaded.')
# In[11]:
def content_loss(base, combination):
return KTF.sum(KTF.square(combination - base))
def yolo_loss(self,
args,
anchors,
num_classes,
ignore_thresh=.5,
print_loss=False):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = self.yolo_head(
yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(
object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(
lambda b, *args: b < m, loop_body, [0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
(1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss
if print_loss:
loss = tf.Print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
K.sum(ignore_mask)
],
message='loss: ')
return loss
def add_invmass_from_8cartesian(x):
return K.concatenate([x, invmass_from_8cartesian(x)])