def yolo_eval(yolo_outputs, anchors, num_classes, image_shape, max_boxes=20, score_threshold=.6, iou_threshold=.5): """Evaluate YOLO model on given input and return filtered boxes.""" num_layers = len(yolo_outputs) anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[ 3, 4, 5 ], [1, 2, 3]] # default setting input_shape = K.shape(yolo_outputs[0])[1:3] * 32 # with tf.Session() as sess: # print(sess.run(input_shape[1])) boxes = [] box_scores = [] for l in range(num_layers): _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) mask = box_scores >= score_threshold max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] for c in range(num_classes): # TODO: use keras backend instead of tf. class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) nms_index = tf.image.non_max_suppression(class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False): """Convert final layer features to bounding box parameters.""" num_anchors = len(anchors) # Reshape to batch, height, width, num_anchors, box_params. anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], 1, 1]) grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, 1, 1]) grid = K.concatenate([grid_x, grid_y]) grid = K.cast(grid, K.dtype(feats)) feats = K.reshape( feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5]) # Adjust preditions to each spatial grid point and anchor size. box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast( grid_shape[::-1], K.dtype(feats)) box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast( input_shape[::-1], K.dtype(feats)) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:]) if calc_loss == True: return grid, feats, box_xy, box_wh return box_xy, box_wh, box_confidence, box_class_probs
def test_shape_operations(self): # concatenate xval = np.random.random((4, 3)) xth = KTH.variable(xval) xtf = KTF.variable(xval) yval = np.random.random((4, 2)) yth = KTH.variable(yval) ytf = KTF.variable(yval) zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1)) ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1)) assert zth.shape == ztf.shape assert_allclose(zth, ztf, atol=1e-05) check_single_tensor_operation('reshape', (4, 2), shape=(8, 1)) check_single_tensor_operation('permute_dimensions', (4, 2, 3), pattern=(2, 0, 1)) check_single_tensor_operation('repeat', (4, 1), n=3) check_single_tensor_operation('flatten', (4, 1)) check_single_tensor_operation('batch_flatten', (20, 2, 5)) check_single_tensor_operation('expand_dims', (4, 3), axis=-1) check_single_tensor_operation('expand_dims', (4, 3, 2), axis=1) check_single_tensor_operation('squeeze', (4, 3, 1), axis=2) check_single_tensor_operation('squeeze', (4, 1, 1), axis=1) check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)}, 'squeeze', {'axis': 2}, (4, 3, 1, 1))
def test_shape_operations(self): # concatenate xval = np.random.random((4, 3)) xth = KTH.variable(xval) xtf = KTF.variable(xval) yval = np.random.random((4, 2)) yth = KTH.variable(yval) ytf = KTF.variable(yval) zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1)) ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1)) assert zth.shape == ztf.shape assert_allclose(zth, ztf, atol=1e-05) check_single_tensor_operation('reshape', (4, 2), shape=(8, 1)) check_single_tensor_operation('permute_dimensions', (4, 2, 3), pattern=(2, 0, 1)) check_single_tensor_operation('repeat', (4, 1), n=3) check_single_tensor_operation('flatten', (4, 1)) check_single_tensor_operation('expand_dims', (4, 3), dim=-1) check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1) check_single_tensor_operation('squeeze', (4, 3, 1), axis=2) check_single_tensor_operation('squeeze', (4, 1, 1), axis=1) check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)}, 'squeeze', {'axis': 2}, (4, 3, 1, 1))
def build_discriminator(self): model = Sequential() ## Head if self.add_invmass_disc: model.add( Lambda( add_invmass_from_8cartesian, input_shape=self.output_shape, output_shape=(self.output_shape[0] + 1, ), )) else: model.add(Dense(128, input_shape=self.output_shape)) if self.do_batch_normalization_disc: model.add(BatchNormalization()) # model.add(LeakyReLU(alpha=0.2)) if self.do_concatenate_disc: model.add(Lambda(lambda x: K.concatenate([x * x, x]))) model.add(LeakyReLU(alpha=0.2)) ## Main Body if self.depth_disc > 0 and self.width_disc > 0: for level in xrange(0, self.depth_disc): model.add(Dense( width_disc / (2**level))) #Triangle with width halved at each level model.add(LeakyReLU(alpha=0.2)) else: model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(64)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(32)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(16)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(8)) model.add(LeakyReLU(alpha=0.2)) ## Tail model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.output_shape) validity = model(img) return Model(img, validity)
def TSNs_SpatialStream(input_shape, classes, num_segments=3, base_model='Xception', dropout_prob=0.8, consensus_type='avg', partial_bn=True): """ Spatial stream of the Temporal Segment Networks (https://arxiv.org/pdf/1705.02953.pdf) defined as multi-input Keras model. """ # Define the shared layers, base conv net and enable partial batch # normalization strategy inputs = [Input(input_shape) for _ in range(num_segments)] dropout = Dropout(dropout_prob) dense = Dense(classes, activation=None) act = Activation(activation='softmax', name='prediction') models_dict = dict(getmembers(keras.applications, isfunction)) base = models_dict[base_model](include_top=False, pooling='avg') if partial_bn: num_bn_layers = 0 for layer in base.layers: if isinstance(layer, BatchNormalization): num_bn_layers += 1 if num_bn_layers != 1: layer.trainable = False # Pass multiple inputs (depending on num_segments) through the base conv # net outputs = [] visual_features = [] for seg_input in inputs: seg_output = base(seg_input) visual_features.append(seg_output) seg_output = dropout(seg_output) seg_output = dense(seg_output) outputs.append(seg_output) # Use a consensus function to combine class scores if consensus_type == 'avg': output = Average()(outputs) elif consensus_type == 'max': output = Maximum()(outputs) elif consensus_type == 'attention': weighted_outputs = [] attn_layer = Dense(1, use_bias=False, name='attn_layer') attn_weights = [attn_layer(_) for _ in visual_features] attn_weights = Lambda(lambda x: K.concatenate(x, axis=-1), name='concatenate')(attn_weights) attn_weights = Activation('softmax')(attn_weights) for i, seg_output in enumerate(outputs): weight = Lambda(lambda x: x[:, i])(attn_weights) weighted_outputs.append(Multiply()([weight, seg_output])) output = Add()(weighted_outputs) output = act(output) model = Model(inputs, output) return model
def yolo_boxes_to_corners(graph, box_xy, box_wh): with graph.as_default(): box_mins = box_xy - (box_wh / 2.) box_maxes = box_xy + (box_wh / 2.) return K.concatenate([ box_mins[..., 1:2], # y_min box_mins[..., 0:1], # x_min box_maxes[..., 1:2], # y_max box_maxes[..., 0:1] # x_max ])
def fix_outputs(x): """ Take nominal delphes format of 19 columns and fix some columns """ return K.concatenate([ # x[:,0:21], x[:, 0:7], # epxpypz for lep1,lep2 -1 for no py2 x[:, 7:8], # nvtx K.sign(x[:, 8:10]), # q1 q2 x[:, 10:12], # iso1 iso2 x[:, 12:14], # met, metphi x[:, 14:19], # jet pts ])
def yolo_correct_boxes(self, box_xy, box_wh, input_shape, image_shape): '''Get corrected boxes''' box_yx = box_xy[..., ::-1] box_hw = box_wh[..., ::-1] input_shape = K.cast(input_shape, K.dtype(box_yx)) image_shape = K.cast(image_shape, K.dtype(box_yx)) new_shape = K.round(image_shape * K.min(input_shape / image_shape)) offset = (input_shape - new_shape) / 2. / input_shape scale = input_shape / new_shape box_yx = (box_yx - offset) * scale box_hw *= scale box_mins = box_yx - (box_hw / 2.) box_maxes = box_yx + (box_hw / 2.) boxes = K.concatenate([ box_mins[..., 0:1], # y_min box_mins[..., 1:2], # x_min box_maxes[..., 0:1], # y_max box_maxes[..., 1:2] # x_max ]) # Scale boxes back to original image shape. boxes *= K.concatenate([image_shape, image_shape]) return boxes
def fix_outputs(x): """ Take nominal delphes format of 21 columns and fix some columns """ return K.concatenate([ # x[:,0:21], x[:, 0:8], # epxpypz for lep1,lep2 K.sign(x[:, 8:9]), # lep1 charge x[:, 9:10], # lep1 iso K.sign(x[:, 10:11]), # lep2 charge x[:, 11:12], # lep2 iso K.round(x[:, 12:13]), # nvtxs x[:, 13:15], # met, metphi K.round(x[:, 15:16]), # ngenjets x[:, 16:21], # jet pts ])
def test_shape_operations(self): # concatenate xval = np.random.random((4, 3)) xth = KTH.variable(xval) xtf = KTF.variable(xval) yval = np.random.random((4, 2)) yth = KTH.variable(yval) ytf = KTF.variable(yval) zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1)) ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1)) assert zth.shape == ztf.shape assert_allclose(zth, ztf, atol=1e-05) check_single_tensor_operation("reshape", (4, 2), shape=(8, 1)) check_single_tensor_operation("permute_dimensions", (4, 2, 3), pattern=(2, 0, 1)) check_single_tensor_operation("repeat", (4, 1), n=3) check_single_tensor_operation("flatten", (4, 1)) check_single_tensor_operation("expand_dims", (4, 3), dim=-1) check_single_tensor_operation("expand_dims", (4, 3, 2), dim=1) check_single_tensor_operation("squeeze", (4, 3, 1), axis=2)
def build_generator(self): model = Sequential() ## Head model.add(Dense(64, input_shape=self.noise_shape)) if self.do_batch_normalization_gen: model.add(BatchNormalization()) model.add(LeakyReLU(alpha=0.2)) if self.do_concatenate_gen: model.add(Lambda(lambda x: K.concatenate([x * x, x]))) model.add(LeakyReLU(alpha=0.2)) ## Main Body if self.depth_gen > 0 and self.width_gen > 0: for level in xrange(0, self.depth_gen): model.add(Dense( width_gen / (2**level))) #Triangle with width halved at each level model.add(LeakyReLU(alpha=0.2)) elif self.beefy_generator: model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) else: model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(128)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(64)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(32)) model.add(LeakyReLU(alpha=0.2)) ## Tail model.add(Dense(self.output_shape[0])) if self.do_tanh_gen: model.add(Activation("tanh")) elif self.fix_delphes_outputs: model.add( Lambda(fix_outputs, input_shape=self.output_shape, output_shape=self.output_shape)) model.summary() noise = Input(shape=self.noise_shape) img = model(noise) return Model(noise, img)
def step(self, a, states): r_tm1 = states[:self.nb_layers] c_tm1 = states[self.nb_layers:2 * self.nb_layers] e_tm1 = states[2 * self.nb_layers:3 * self.nb_layers] if self.extrap_start_time is not None: t = states[-1] a = K.switch( t >= self.t_extrap, states[-2], a ) # if past self.extrap_start_time, the previous prediction will be treated as the actual c = [] r = [] e = [] # Update R units starting from the top for l in reversed(range(self.nb_layers)): inputs = [r_tm1[l], e_tm1[l]] if l < self.nb_layers - 1: inputs.append(r_up) inputs = K.concatenate(inputs, axis=self.channel_axis) i = self.conv_layers['i'][l].call(inputs) f = self.conv_layers['f'][l].call(inputs) o = self.conv_layers['o'][l].call(inputs) _c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs) _r = o * self.LSTM_activation(_c) c.insert(0, _c) r.insert(0, _r) if l > 0: r_up = self.upsample.call(_r) # Update feedforward path starting from the bottom for l in range(self.nb_layers): ahat = self.conv_layers['ahat'][l].call(r[l]) if l == 0: ahat = K.minimum(ahat, self.pixel_max) frame_prediction = ahat # compute errors e_up = self.error_activation(ahat - a) e_down = self.error_activation(a - ahat) e.append(K.concatenate((e_up, e_down), axis=self.channel_axis)) if self.output_layer_num == l: if self.output_layer_type == 'A': output = a elif self.output_layer_type == 'Ahat': output = ahat elif self.output_layer_type == 'R': output = r[l] elif self.output_layer_type == 'E': output = e[l] if l < self.nb_layers - 1: a = self.conv_layers['a'][l].call(e[l]) a = self.pool.call(a) # target for next layer if self.output_layer_type is None: if self.output_mode == 'prediction': output = frame_prediction else: for l in range(self.nb_layers): layer_error = K.mean(K.batch_flatten(e[l]), axis=-1, keepdims=True) all_error = layer_error if l == 0 else K.concatenate( (all_error, layer_error), axis=-1) if self.output_mode == 'error': output = all_error else: output = K.concatenate( (K.batch_flatten(frame_prediction), all_error), axis=-1) states = r + c + e if self.extrap_start_time is not None: states += [frame_prediction, t + 1] return output, states
def add_invmass_from_8cartesian(x): invmass = K.sqrt((x[:, 0:1] + x[:, 4:5])**2 - (x[:, 1:2] + x[:, 5:6])**2 - (x[:, 2:3] + x[:, 6:7])**2 - (x[:, 3:4] + x[:, 7:8])**2) return K.concatenate([x, invmass])
# this will contain our generated image if K.image_dim_ordering() == 'th': combination_image = K.placeholder((1, 3, img_width, img_height)) else: combination_image = K.placeholder((1, img_width, img_height, 3)) image_tensor = [base_image] for style_image_tensor in style_reference_images: image_tensor.append(style_image_tensor) image_tensor.append(combination_image) nb_tensors = len(image_tensor) nb_style_images = nb_tensors - 2 # Content andoutput image not considered # combine the various image into a single Keras tensor input_tensor = K.concatenate(image_tensor, axis=0) if K.image_dim_ordering() == 'th': shape = (nb_tensors, 3, img_width, img_height) else: shape = (nb_tensors, img_width, img_height, 3) ip = Input(tensor=input_tensor, shape=shape) # build a network x = Conv2D(64, (3, 3), activation='relu', name='conv1_1', padding='same')(ip) x = Conv2D(64, (3, 3), activation='relu', name='conv1_2', padding='same')(x) x = pooling_func(x) x = Conv2D(128, (3, 3), activation='relu', name='conv2_1', padding='same')(x) x = Conv2D(128, (3, 3), activation='relu', name='conv2_2', padding='same')(x)
return tf.Session(config=tf.ConfigProto( gpu_options=gpu_options, intra_op_parallelism_threads=num_threads)) else: return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) import keras.backend.tensorflow_backend as KTF KTF.set_session(get_session()) target_image = KTF.constant(preprocess_image(target_image_path)) style_reference_image = KTF.constant( preprocess_image(style_reference_image_path)) combination_image = KTF.placeholder((1, img_height, img_width, 3)) # In[9]: input_tensor = KTF.concatenate( [target_image, style_reference_image, combination_image], axis=0) # In[10]: model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) print('Model loaded.') # In[11]: def content_loss(base, combination): return KTF.sum(KTF.square(combination - base))
def yolo_loss(self, args, anchors, num_classes, ignore_thresh=.5, print_loss=False): '''Return yolo_loss tensor Parameters ---------- yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body y_true: list of array, the output of preprocess_true_boxes anchors: array, shape=(N, 2), wh num_classes: integer ignore_thresh: float, the iou threshold whether to ignore object confidence loss Returns ------- loss: tensor, shape=(1,) ''' num_layers = len(anchors) // 3 # default setting yolo_outputs = args[:num_layers] y_true = args[num_layers:] anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2] ] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]] input_shape = K.cast( K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0])) grid_shapes = [ K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers) ] loss = 0 m = K.shape(yolo_outputs[0])[0] # batch size, tensor mf = K.cast(m, K.dtype(yolo_outputs[0])) for l in range(num_layers): object_mask = y_true[l][..., 4:5] true_class_probs = y_true[l][..., 5:] grid, raw_pred, pred_xy, pred_wh = self.yolo_head( yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, calc_loss=True) pred_box = K.concatenate([pred_xy, pred_wh]) # Darknet raw box to calculate loss. raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1]) raw_true_wh = K.switch( object_mask, raw_true_wh, K.zeros_like(raw_true_wh)) # avoid log(0)=-inf box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4] # Find ignore mask, iterate over each of batch. ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True) object_mask_bool = K.cast(object_mask, 'bool') def loop_body(b, ignore_mask): true_box = tf.boolean_mask(y_true[l][b, ..., 0:4], object_mask_bool[b, ..., 0]) iou = box_iou(pred_box[b], true_box) best_iou = K.max(iou, axis=-1) ignore_mask = ignore_mask.write( b, K.cast(best_iou < ignore_thresh, K.dtype(true_box))) return b + 1, ignore_mask _, ignore_mask = K.control_flow_ops.while_loop( lambda b, *args: b < m, loop_body, [0, ignore_mask]) ignore_mask = ignore_mask.stack() ignore_mask = K.expand_dims(ignore_mask, -1) # K.binary_crossentropy is helpful to avoid exp overflow. xy_loss = object_mask * box_loss_scale * K.binary_crossentropy( raw_true_xy, raw_pred[..., 0:2], from_logits=True) wh_loss = object_mask * box_loss_scale * 0.5 * K.square( raw_true_wh - raw_pred[..., 2:4]) confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \ (1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask class_loss = object_mask * K.binary_crossentropy( true_class_probs, raw_pred[..., 5:], from_logits=True) xy_loss = K.sum(xy_loss) / mf wh_loss = K.sum(wh_loss) / mf confidence_loss = K.sum(confidence_loss) / mf class_loss = K.sum(class_loss) / mf loss += xy_loss + wh_loss + confidence_loss + class_loss if print_loss: loss = tf.Print(loss, [ loss, xy_loss, wh_loss, confidence_loss, class_loss, K.sum(ignore_mask) ], message='loss: ') return loss
def add_invmass_from_8cartesian(x): return K.concatenate([x, invmass_from_8cartesian(x)])