def get_unet_model(IMG_HEIGHT=300, IMG_WIDTH=300, IMG_CHANNELS=3):
inputs = layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
s = Lambda(lambda x: x / 1)(inputs)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(s)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = Dropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = Dropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = Dropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = Dropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = Dropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = Dropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = Dropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[mean_iou])
model.summary()
return model
def model(reader):
inshape = (None, )
known_in = L.Input(shape=inshape, name='known_input', dtype='int32')
unknown_in = L.Input(shape=inshape, name='unknown_input', dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 5)
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
conv8 = L.Convolution1D(filters=500,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8')
conv4 = L.Convolution1D(filters=500,
kernel_size=4,
strides=1,
activation='relu',
name='convolutional_4')
conv16 = L.Convolution1D(filters=200,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8_2')
repr_known1 = L.GlobalMaxPooling1D(name='known_repr_8')(conv8(known_embed))
repr_unknown1 = L.GlobalMaxPooling1D(name='unknown_repr_8')(
conv8(unknown_embed))
repr_known2 = L.GlobalMaxPooling1D(name='known_repr_4')(conv4(known_embed))
repr_unknown2 = L.GlobalMaxPooling1D(name='unknown_repr_4')(
conv4(unknown_embed))
repr_known3 = L.GlobalMaxPooling1D(name='known_repr_8_2')(
conv16(known_embed))
repr_unknown3 = L.GlobalMaxPooling1D(name='unknown_repr_8_2')(
conv16(unknown_embed))
repr_known = L.Concatenate()([repr_known1, repr_known2, repr_known3])
repr_unknown = L.Concatenate()(
[repr_unknown1, repr_unknown2, repr_unknown3])
abs_diff = L.merge(inputs=[repr_known, repr_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
dense3 = L.Dense(500, activation='relu')(dense2)
dense4 = L.Dense(500, activation='relu')(dense3)
pruned = L.Dropout(0.3)(dense4)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def add_prior(input_model,
prior_shape,
name='prior_model',
prefix=None,
use_logp=True,
final_pred_activation='softmax',
add_prior_layer_reg=0):
"""
Append post-prior layer to a given model
"""
# naming
model_name = name
if prefix is None:
prefix = model_name
# prior input layer
prior_input_name = '%s-input' % prefix
prior_tensor = KL.Input(shape=prior_shape, name=prior_input_name)
prior_tensor_input = prior_tensor
like_tensor = input_model.output
# operation varies depending on whether we log() prior or not.
if use_logp:
# name = '%s-log' % prefix
# prior_tensor = KL.Lambda(_log_layer_wrap(add_prior_layer_reg), name=name)(prior_tensor)
print("Breaking change: use_logp option now requires log input!",
file=sys.stderr)
merge_op = KL.add
else:
# using sigmoid to get the likelihood values between 0 and 1
# note: they won't add up to 1.
name = '%s_likelihood_sigmoid' % prefix
like_tensor = KL.Activation('sigmoid', name=name)(like_tensor)
merge_op = KL.multiply
# merge the likelihood and prior layers into posterior layer
name = '%s_posterior' % prefix
post_tensor = merge_op([prior_tensor, like_tensor], name=name)
# output prediction layer
# we use a softmax to compute P(L_x|I) where x is each location
pred_name = '%s_prediction' % prefix
if final_pred_activation == 'softmax':
assert use_logp, 'cannot do softmax when adding prior via P()'
print("using final_pred_activation %s for %s" %
(final_pred_activation, model_name))
softmax_lambda_fcn = lambda x: keras.activations.softmax(x, axis=-1)
pred_tensor = KL.Lambda(softmax_lambda_fcn,
name=pred_name)(post_tensor)
else:
pred_tensor = KL.Activation('linear', name=pred_name)(post_tensor)
# create the model
model_inputs = [*input_model.inputs, prior_tensor_input]
model = Model(inputs=model_inputs, outputs=[pred_tensor], name=model_name)
# compile
return model
def design_dnn(nb_features,
input_shape,
nb_levels,
conv_size,
nb_labels,
feat_mult=1,
pool_size=2,
padding='same',
activation='elu',
final_layer='dense-sigmoid',
conv_dropout=0,
conv_maxnorm=0,
nb_input_features=1,
batch_norm=False,
name=None,
prefix=None,
use_strided_convolution_maxpool=True,
nb_conv_per_level=2):
"""
"deep" cnn with dense or global max pooling layer @ end...
Could use sequential...
"""
model_name = name
if model_name is None:
model_name = 'model_1'
if prefix is None:
prefix = model_name
ndims = len(input_shape)
input_shape = tuple(input_shape)
convL = getattr(KL, 'Conv%dD' % ndims)
maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D
if isinstance(pool_size, int):
pool_size = (pool_size, ) * ndims
# kwargs for the convolution layer
conv_kwargs = {'padding': padding, 'activation': activation}
if conv_maxnorm > 0:
conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm)
# initialize a dictionary
enc_tensors = {}
# first layer: input
name = '%s_input' % prefix
enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ),
name=name)
last_tensor = enc_tensors[name]
# down arm:
# add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers
for level in range(nb_levels):
for conv in range(nb_conv_per_level):
if conv_dropout > 0:
name = '%s_dropout_%d_%d' % (prefix, level, conv)
enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_conv_%d_%d' % (prefix, level, conv)
nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int)
enc_tensors[name] = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
# max pool
if use_strided_convolution_maxpool:
name = '%s_strided_conv_%d' % (prefix, level)
enc_tensors[name] = convL(nb_lvl_feats,
pool_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
else:
name = '%s_maxpool_%d' % (prefix, level)
enc_tensors[name] = maxpool(pool_size=pool_size,
name=name,
padding=padding)(last_tensor)
last_tensor = enc_tensors[name]
# dense layer
if final_layer == 'dense-sigmoid':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name,
activation="sigmoid")(last_tensor)
elif final_layer == 'dense-tanh':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name)(last_tensor)
last_tensor = enc_tensors[name]
# Omittting BatchNorm for now, it seems to have a cpu vs gpu problem
# https://github.com/tensorflow/tensorflow/pull/8906
# https://github.com/fchollet/keras/issues/5802
# name = '%s_%s_bn' % prefix
# enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor)
# last_tensor = enc_tensors[name]
name = '%s_%s_tanh' % prefix
enc_tensors[name] = KL.Activation(activation="tanh",
name=name)(last_tensor)
elif final_layer == 'dense-softmax':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(nb_labels,
name=name,
activation="softmax")(last_tensor)
# global max pooling layer
elif final_layer == 'myglobalmaxpooling':
name = '%s_batch_norm' % prefix
enc_tensors[name] = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool_reshape' % prefix
enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_sigmoid' % prefix
enc_tensors[name] = KL.Conv1D(1,
1,
name=name,
activation="sigmoid",
use_bias=True)(last_tensor)
elif final_layer == 'globalmaxpooling':
name = '%s_conv_to_featmaps' % prefix
enc_tensors[name] = KL.Conv3D(2, 1, name=name,
activation="relu")(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_softmax' % prefix
enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor)
last_tensor = enc_tensors[name]
# create the model
model = Model(inputs=[enc_tensors['%s_input' % prefix]],
outputs=[last_tensor],
name=model_name)
return model
def VGG19_l2(input_shape=None, classes=5, use_soft=True):
img_input = layers.Input(shape=input_shape)
# Block 1
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer="he_normal")(img_input)
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# Classification block
x = layers.Flatten(name='flatten')(x)
x = layers.Dense(512,
activation='relu',
name='fc1',
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4),
activity_regularizer=regularizers.l2(1e-4))(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(128,
activation='relu',
name='fc2',
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4),
activity_regularizer=regularizers.l2(1e-4))(x)
#x=layers.Dropout(0.8)(x)
if use_soft:
x = Dense(classes,
activation="softmax",
name='predictions',
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4),
activity_regularizer=regularizers.l2(1e-4))(x)
else:
x = Dense(classes,
activation="linear",
name="Z_4",
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4),
activity_regularizer=regularizers.l2(1e-4))(x)
model = models.Model(img_input, x, name='vgg19')
return model
# 验证
# input_shape越大，精度会上升，但速度会下降。
# input_shape = (320, 320)
# input_shape = (416, 416)
input_shape = (608, 608)
# 验证时的分数阈值和nms_iou阈值
conf_thresh = 0.001
nms_thresh = 0.45
# 是否画出验证集图片
draw_image = False
# 验证时的批大小
eval_batch_size = 4
# 多尺度训练
inputs = layers.Input(shape=(None, None, 3))
model_body = YOLOv4(inputs, num_classes, num_anchors)
_decode = Decode(conf_thresh, nms_thresh, input_shape, model_body, class_names)
# 模式。 0-从头训练，1-读取之前的模型继续训练（model_path可以是'yolov4.h5'、'./weights/step00001000.h5'这些。）
pattern = 1
max_bbox_per_scale = 150
iou_loss_thresh = 0.7
if pattern == 1:
lr = 0.0001
batch_size = 8
model_path = 'yolov4.h5'
# model_path = './weights/step00001000.h5'
model_body.load_weights(model_path, by_name=True)
strs = model_path.split('step')
if len(strs) == 2:
def conv_enc(nb_features,
input_shape,
nb_levels,
conv_size,
name=None,
prefix=None,
feat_mult=1,
pool_size=2,
dilation_rate_mult=1,
padding='same',
activation='elu',
layer_nb_feats=None,
use_residuals=False,
nb_conv_per_level=2,
conv_dropout=0,
batch_norm=None):
"""
Fully Convolutional Encoder
"""
# naming
model_name = name
if prefix is None:
prefix = model_name
# volume size data
ndims = len(input_shape) - 1
input_shape = tuple(input_shape)
if isinstance(pool_size, int):
pool_size = (pool_size, ) * ndims
# prepare layers
convL = getattr(KL, 'Conv%dD' % ndims)
conv_kwargs = {'padding': padding, 'activation': activation}
maxpool = getattr(KL, 'MaxPooling%dD' % ndims)
# first layer: input
name = '%s_input' % prefix
last_tensor = KL.Input(shape=input_shape, name=name)
input_tensor = last_tensor
# down arm:
# add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers
lfidx = 0
for level in range(nb_levels):
lvl_first_tensor = last_tensor
nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int)
conv_kwargs['dilation_rate'] = dilation_rate_mult**level
for conv in range(nb_conv_per_level):
if layer_nb_feats is not None:
nb_lvl_feats = layer_nb_feats[lfidx]
lfidx += 1
name = '%s_conv_downarm_%d_%d' % (prefix, level, conv)
if conv < (nb_conv_per_level - 1) or (not use_residuals):
last_tensor = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(last_tensor)
else: # no activation
last_tensor = convL(nb_lvl_feats,
conv_size,
padding=padding,
name=name)(last_tensor)
if conv_dropout > 0:
# conv dropout along feature space only
name = '%s_dropout_downarm_%d_%d' % (prefix, level, conv)
noise_shape = [None, *[1] * ndims, nb_lvl_feats]
last_tensor = KL.Dropout(conv_dropout,
noise_shape=noise_shape)(last_tensor)
if use_residuals:
convarm_layer = last_tensor
# the "add" layer is the original input
# However, it may not have the right number of features to be added
nb_feats_in = lvl_first_tensor.get_shape()[-1]
nb_feats_out = convarm_layer.get_shape()[-1]
add_layer = lvl_first_tensor
if nb_feats_in > 1 and nb_feats_out > 1 and (nb_feats_in !=
nb_feats_out):
name = '%s_expand_down_merge_%d' % (prefix, level)
last_tensor = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(lvl_first_tensor)
add_layer = last_tensor
if conv_dropout > 0:
name = '%s_dropout_down_merge_%d_%d' % (prefix, level,
conv)
noise_shape = [None, *[1] * ndims, nb_lvl_feats]
last_tensor = KL.Dropout(
conv_dropout, noise_shape=noise_shape)(last_tensor)
name = '%s_res_down_merge_%d' % (prefix, level)
last_tensor = KL.add([add_layer, convarm_layer], name=name)
name = '%s_res_down_merge_act_%d' % (prefix, level)
last_tensor = KL.Activation(activation, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_bn_down_%d' % (prefix, level)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# max pool if we're not at the last level
if level < (nb_levels - 1):
name = '%s_maxpool_%d' % (prefix, level)
last_tensor = maxpool(pool_size=pool_size,
name=name,
padding=padding)(last_tensor)
# create the model and return
model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name)
return model
def model(self, model_type):
input_image = KL.Input(shape=[None, None, 3], name='input_image')
input_bbox = KL.Input(shape=[None, 4], name='input_bbox')
input_image_meta = KL.Input(shape=[self.cfg.IMAGE_META_SIZE],
name="input_image_meta")
input_anchors = KL.Input(shape=[None, 4], name="input_anchors")
detection_map, reid_map = self.reid(input_image)
rpn = self.build_rpn_model(self.cfg.RPN_ANCHOR_STRIDE,
len(self.cfg.RPN_ANCHOR_RATIOS),
self.cfg.TOP_DOWN_PYRAMID_SIZE)
outputs = list(zip(*[rpn(p) for p in detection_map]))
rpn_class_logits, rpn_class, rpn_bbox = [
KL.Concatenate(axis=1, name=n)(list(o)) for o, n in zip(
outputs, ["rpn_class_logits", "rpn_class", "rpn_bbox"])
]
rpn_rois = ProposalLayer(
proposal_count=self.cfg.POST_NMS_ROIS_INFERENCE,
nms_threshold=self.cfg.RPN_NMS_THRESHOLD,
name="ROI",
config=self.cfg)([rpn_class, rpn_bbox, input_anchors])
rois = KL.Lambda(lambda x: tf.concat(x, axis=1))(
[input_bbox, rpn_rois])
x = PyramidROIAlign(
[self.cfg.POOL_SIZE, self.cfg.POOL_SIZE],
name="roi_align_classifier")([rois, input_image_meta] +
detection_map[:-1])
x = self.mrcnn_feature(x)
scores, bboxes = self.mrcnn_box_reg([rois, x])
detection_scores, regression_scores, detection, regression = KL.Lambda(lambda x : [x[0][:, tf.shape(input_bbox)[1]:, :], \
x[0][:, :tf.shape(input_bbox)[1], :], \
x[1][:, tf.shape(input_bbox)[1]:, :], \
x[1][:, :tf.shape(input_bbox)[1], :]
])([scores, bboxes])
detection_scores, detection = KL.Lambda(
lambda x: self.nms_selection(x[0], x[1]))(
[detection_scores, detection])
if model_type == 'detection':
return KM.Model(
[input_image, input_bbox, input_image_meta, input_anchors],
[detection, detection_scores],
name='mrcnn')
bboxes = KL.Lambda(lambda x: tf.concat(x, axis=1))(
[input_bbox, regression, detection])
reid_map = ATLnet(reid_map, layer=self.cfg.layer, SEnet=self.cfg.SEnet)
pooled = feature_pooling(self.cfg,
name='alignedROIPooling')([bboxes] + reid_map)
pooled = KL.Lambda(lambda x: tf.squeeze(x, axis=0))(pooled)
vectors = sMGN(pooled,
_eval=True,
return_all=self.cfg.mgn,
return_mgn=True,
l2_norm=self.cfg.l2_norm)
vectors = KL.Lambda(lambda x: tf.expand_dims(x[:, 0, 0, :], axis=0))(
vectors)
prediction_vector, regression_vector, detection_vector = KL.Lambda(lambda x : [x[:, :tf.shape(input_bbox)[1], :], \
x[:, tf.shape(input_bbox)[1]:(2*tf.shape(input_bbox)[1]), :], \
x[:, (2*tf.shape(input_bbox)[1]):, :], \
])(vectors)
return KM.Model(
[input_image, input_bbox, input_image_meta, input_anchors], [
prediction_vector, regression_vector, regression,
regression_scores, detection_vector, detection,
detection_scores
])
def build_rpn_model(self, anchor_stride, anchors_per_location, depth):
input_feature_map = KL.Input(shape=[None, None, depth],
name="input_rpn_feature_map")
outputs = self.rpn_graph(input_feature_map, anchors_per_location,
anchor_stride)
return KM.Model([input_feature_map], outputs, name="rpn_model")
# import ipdb; ipdb.set_trace()
# train dataset
dataset_train = u.get_dataset(coco_path, 'train')
gen_train = prepare(dataset_train, epochs, batch_size, input_shape,
output_shape)
callback = ModelSaveBestAvgAcc(filepath="model-{epoch:02d}-{avgacc:.2f}.hdf5",
verbose=True,
cond=filter_val('fmeasure'))
losses = []
for i in range(0, 2):
losses.append(binary_focal_loss(gamma=2.))
input_tensor = layers.Input(shape=input_shape)
model = get_model(input_tensor=input_tensor)
outputs = model.outputs
x = layers.Multiply()([input_tensor, model.output])
x = model(x)
# import ipdb; ipdb.set_trace()
model = models.Model(inputs=input_tensor, outputs=outputs + [x])
model.compile(optimizer=opt.Adam(lr=1e-4),
loss=losses,
metrics=['accuracy', precision, recall, fmeasure])
model.summary()
def ResNet(stack_fn,
preact,
use_bias,
model_name='resnet',
include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
pooling=None,
nclass=1000,
**kwargs):
"""Instantiates the ResNet, ResNetV2, and ResNeXt architecture.
Optionally loads weights pre-trained on ImageNet.
Note that the data format convention used by the model is
the one specified in your Keras config at `~/.keras/keras.json`.
# Arguments
stack_fn: a function that returns output tensor for the
stacked residual blocks.
preact: whether to use pre-activation or not
(True for ResNetV2, False for ResNet and ResNeXt).
use_bias: whether to use biases for convolutional layers or not
(True for ResNet and ResNetV2, False for ResNeXt).
model_name: string, model name.
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor
(i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
# Determine proper input shape
# input_shape = input_shape
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input)
x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x)
if preact is False:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='conv1_bn')(x)
x = layers.Activation('relu', name='conv1_relu')(x)
x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x)
x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x)
x = stack_fn(x)
if preact is True:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='post_bn')(x)
x = layers.Activation('relu', name='post_relu')(x)
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(nclass, activation='softmax', name='probs')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D(name='max_pool')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = models.Model(inputs, x, name=model_name)
# Load weights.
if weights is not None:
model.load_weights(weights)
model.compile(loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'])
return model
def CapsNet(input_shape, n_class, num_routing):
from keras import layers, models
from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask
from keras.preprocessing import sequence
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 4d, [None, width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=(maxlen, ), dtype='int32')
embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x)
# conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(embed)
conv1 = layers.Conv1D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(embed)
pool = layers.MaxPooling1D(pool_size=231, strides=1)(conv1)
# lstm = layers.LSTM(9, return_sequences=True, recurrent_dropout=0.6, dropout = 0.9, name='lstm1')(conv1)
# lstm = layers.CuDNNLSTM(9, return_sequences=True, name='lstm1')(pool)
# lstm2 = layers.CuDNNLSTM(9, return_sequences=True, name='lstm2')(lstm)
# drop = layers.Dropout(0.5, name='drop1')(lstm)
# td = layers.TimeDistributed(layers.Dense(81, activation='softmax'))(lstm)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
# primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
primarycaps = PrimaryCap(pool,
dim_vector=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
# digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps)
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_vector=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked = Mask()(
[digitcaps,
y]) # The true label is used to mask the output of capsule layer.
# x_recon = layers.Dense(512, activation='relu')(masked)
# x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(512, activation='relu')(masked)
# x_recon = layers.Dropout(0.9)(x_recon)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
# x_recon = layers.Dropout(0.9)(x_recon)
x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon)
# x_recon = layers.Reshape(target_shape=[1], name='out_recon')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
##################
train_dataset = '../../200_dataset/norm_audio_video/tr_set.hdf5'
val_dataset = '../../200_dataset/norm_audio_video/val_set.hdf5'
test_dataset = '../../200_dataset/norm_audio_video/test_set.hdf5'
batch_size = 10
epochs = 100
train_generator = data_generator(train_dataset, batch_size)
val_generator = data_generator(val_dataset, batch_size)
##V:video1 VV:video2
V1 = layers.Input(shape=(75, 512), name='Video1_input')
V2 = layers.Conv1D(256,
kernel_size=7,
dilation_rate=1,
padding='same',
activation='relu')(V1)
V3 = layers.BatchNormalization(axis=-1)(V2)
V4 = layers.Conv1D(256,
kernel_size=5,
dilation_rate=1,
padding='same',
activation='relu')(V3)
V5 = layers.BatchNormalization(axis=-1)(V4)
V6 = layers.Conv1D(256,
kernel_size=5,
dilation_rate=2,
def get_model(img_size=(1, 300, 300, 3), num_classes=4):
inputs = layers.Input(img_size, name='RGB_Input')
print('input shape ', inputs.shape)
### [First half of the network: downsampling inputs] ###
# Entry block
x = layers.Conv2D(32, 3, strides=2, padding="same")(inputs)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
previous_block_activation = x # Set aside residual
# Blocks 1, 2, 3 are identical apart from the feature depth.
for filters in [64, 128, 256]:
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPooling2D(3, strides=2, padding="same")(x)
# Project residual
residual = layers.Conv2D(filters, 1, strides=2,
padding="same")(previous_block_activation)
x = layers.add([x, residual]) # Add back residual
previous_block_activation = x # Set aside next residual
### [Second half of the network: upsampling inputs] ###
for filters in [256, 128, 64, 32]:
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.UpSampling2D(2)(x)
# Project residual
residual = layers.UpSampling2D(2)(previous_block_activation)
residual = layers.Conv2D(filters, 1, padding="same")(residual)
x = layers.add([x, residual]) # Add back residual
previous_block_activation = x # Set aside next residual
# Add a per-pixel classification layer
#outputs = layers.Conv2D(num_classes, 0, activation="softmax", padding="same")(x)
d = layers.Convolution2D(1, (1, 1), activation='sigmoid',
padding='same')(x)
d = layers.Cropping2D((EDGE_CROP, EDGE_CROP))(d)
d = layers.ZeroPadding2D((EDGE_CROP, EDGE_CROP))(d)
# Define the model
model0 = models.Model(inputs=[inputs], outputs=[d])
#model = models.Model(inputs, outputs)
return model0
# In[6]:
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
# In[10]:
import keras.layers as layers
from sklearn import metrics
metrics.classification
layers.Input()
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=15,
first_model = models.Model(inputs=model1.input, outputs=first_model)
second_model = model2.layers[9].output
second_model = models.Model(inputs=model2.input, outputs=second_model)
first_model.summary()
second_model.summary()
pred1_train = first_model.predict(x_train)
pred1_test = first_model.predict(x_test)
pred2_train = second_model.predict(x_train)
pred2_test = second_model.predict(x_test)
x_train = np.column_stack((pred1_train, pred2_train))
x_test = np.column_stack((pred1_test, pred2_test))
print(x_train.shape)
print(x_test.shape)
input_audio = layers.Input(shape=(x_train.shape[1], ))
model = layers.Dense(32, activation='relu')(input_audio)
#model = layers.Dense(32, activation='relu')(model)
model = layers.Dense(3, activation='softmax')(model)
model = Model(input_audio, model)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
model.summary()
path_save = os.path.join("/users/home/s18023/DCASE2020/taskb/final/model_d")
os.chdir(path_save)
checkpointer = ModelCheckpoint(filepath=weightpath_name,
monitor='val_loss',
def Xception(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):
"""Instantiates the Xception architecture.
Optionally loads weights pre-trained on ImageNet.
Note that the data format convention used by the model is
the one specified in your Keras config at `~/.keras/keras.json`.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor
(i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)`.
It should have exactly 3 inputs channels,
and width and height should be no smaller than 71.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional block.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional block, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True,
and if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
# backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=71,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
x = layers.Conv2D(32, (3, 3),
strides=(2, 2),
use_bias=False,
name='block1_conv1')(img_input)
x = layers.BatchNormalization(axis=channel_axis, name='block1_conv1_bn')(x)
x = layers.Activation('relu', name='block1_conv1_act')(x)
x = layers.Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block1_conv2_bn')(x)
x = layers.Activation('relu', name='block1_conv2_act')(x)
residual = layers.Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv1_bn')(x)
x = layers.Activation('relu', name='block2_sepconv2_act')(x)
x = layers.SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv2_bn')(x)
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block2_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(256, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block3_sepconv1_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv1_bn')(x)
x = layers.Activation('relu', name='block3_sepconv2_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv2_bn')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2),
padding='same',
name='block3_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(728, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block4_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv1_bn')(x)
x = layers.Activation('relu', name='block4_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv2_bn')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2),
padding='same',
name='block4_pool')(x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = layers.Activation('relu', name=prefix + '_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv1_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv2_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv3_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv3')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv3_bn')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(1024, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block13_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv1_bn')(x)
x = layers.Activation('relu', name='block13_sepconv2_act')(x)
x = layers.SeparableConv2D(1024, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv2_bn')(x)
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block13_pool')(x)
x = layers.add([x, residual])
x = layers.SeparableConv2D(1536, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv1_bn')(x)
x = layers.Activation('relu', name='block14_sepconv1_act')(x)
x = layers.SeparableConv2D(2048, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv2_bn')(x)
x = layers.Activation('relu', name='block14_sepconv2_act')(x)
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = models.Model(inputs, x, name='xception')
# Load weights.
if weights == 'imagenet':
if include_top:
weights_path = keras_utils.get_file(
'xception_weights_tf_dim_ordering_tf_kernels.h5',
TF_WEIGHTS_PATH,
cache_subdir='models',
file_hash='0a58e3b7378bc2990ea3b43d5981f1f6')
else:
weights_path = keras_utils.get_file(
'xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='b0042744bf5b25fce3cb969f33bebb97')
model.load_weights(weights_path)
if backend.backend() == 'theano':
keras_utils.convert_all_kernels_in_model(model)
elif weights is not None:
model.load_weights(weights)
return model
x = layers.LeakyReLU()(x) x = layers.Reshape((6, 6, 128))(x) x = layers.Conv2D(256, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2DTranspose(256, 4, strides=1, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(256, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(channels, 7, activation='tanh', padding='same')(x) generator = keras.models.Model(generator_input, x) generator.summary() discriminator_input = layers.Input(shape=(height, width, channels)) x = layers.Conv2D(128, 1)(discriminator_input) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Flatten()(x) x = layers.Dropout(0, 4)(x) x = layers.Dense(1, activation='sigmoid')(x) discriminator = keras.models.Model(discriminator_input, x) discriminator.summary()
def main():
# user options
batch_size = 128
val_in_train = False # not sure how the validation part works during fit.
use_model_checkpt = False
# demo processing
sess = tf.Session()
KB.set_session(sess)
gdev_list = get_available_gpus()
ngpus = len(gdev_list)
batch_size = batch_size * ngpus
data = mnist.load_mnist()
X_train = data.train.images
# X_test = data.test.images
train_samples = X_train.shape[0] # 60000
# test_samples = X_test.shape[0] # 10000
height_nrows = 28
width_ncols = 28
batch_shape = [batch_size, height_nrows, width_ncols, 1]
epochs = 5
steps_per_epoch = train_samples // batch_size
# validations_steps = test_samples / batch_size
nclasses = 10
# The capacity variable controls the maximum queue size
# allowed when prefetching data for training.
capacity = 10000
# min_after_dequeue is the minimum number elements in the queue
# after a dequeue, which ensures sufficient mixing of elements.
min_after_dequeue = 3000
# If `enqueue_many` is `False`, `tensors` is assumed to represent a
# single example. An input tensor with shape `[x, y, z]` will be output
# as a tensor with shape `[batch_size, x, y, z]`.
#
# If `enqueue_many` is `True`, `tensors` is assumed to represent a
# batch of examples, where the first dimension is indexed by example,
# and all members of `tensors` should have the same size in the
# first dimension. If an input tensor has shape `[*, x, y, z]`, the
# output will have shape `[batch_size, x, y, z]`.
enqueue_many = True
x_train_batch, y_train_batch = tf.train.shuffle_batch(
tensors=[data.train.images, data.train.labels.astype(np.int32)],
batch_size=batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue,
enqueue_many=enqueue_many,
num_threads=8)
x_train_batch = tf.cast(x_train_batch, tf.float32)
x_train_batch = tf.reshape(x_train_batch, shape=batch_shape)
y_train_batch = tf.cast(y_train_batch, tf.int32)
y_train_batch = tf.one_hot(y_train_batch, nclasses)
x_train_input = Input(tensor=x_train_batch)
# x_test_batch, y_test_batch = tf.train.batch(
# tensors=[data.test.images, data.test.labels.astype(np.int32)],
# batch_size=batch_size,
# capacity=capacity,
# enqueue_many=enqueue_many,
# num_threads=8)
# I like the non-functional definition of model more.
# model_init = make_model(x_train_input, nclasses)
# x_train_out = model_init.output
# train_model = Model(inputs=[x_train_input], outputs=[x_train_out])
x_train_out = cnn_layers(x_train_input, nclasses)
train_model = Model(inputs=[x_train_input], outputs=[x_train_out])
if ngpus > 1:
train_model = make_parallel(train_model, gdev_list)
lr = 2e-3 * ngpus
train_model.compile(optimizer=RMSprop(lr=lr, decay=1e-5),
loss='categorical_crossentropy',
metrics=['accuracy'],
target_tensors=[y_train_batch])
if ngpus > 1:
print_mgpu_modelsummary(train_model)
else:
train_model.summary()
# Callbacks
if use_model_checkpt:
mon = 'val_acc' if val_in_train else 'acc'
checkpoint = ModelCheckpoint(
'saved_wt.h5', monitor=mon, verbose=0,
save_best_only=True,
save_weights_only=True)
checkpoint = [checkpoint]
else:
checkpoint = []
callbacks = checkpoint
# Training slower with callback. Multigpu slower with callback during
# training than 1 GPU. Again, mnist is too trivial of a model and dataset
# to benchmark or stress GPU compute capabilities. I set up this example
# to illustrate potential for speedup of multigpu case trying to use mnist
# as a stressor.
# It's like comparing a 5 ft race between a person and a truck. A truck is
# obviously faster than a person but in a 5 ft race the person will likely
# win due to slower startup for the truck.
# I will re-implement this with Cifar that should be a better benchmark.
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
# Fit the model using data from the TFRecord data tensors.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
start_time = time.time()
train_model.fit(
# validation_data=(x_test_batch, y_test_batch)
# if val_in_train else None, # validation data is not used???
# validations_steps if val_in_train else None,
# validation_steps=val_in_train,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks)
elapsed_time = time.time() - start_time
print('[{}] finished in {} s'.format('TRAINING', round(elapsed_time, 3)))
if not checkpoint: # empty list
train_model.save_weights('./saved_wt.h5')
# Clean up the TF session.
coord.request_stop()
coord.join(threads)
KB.clear_session()
# Second Session. Demonstrate that the model works and is independent of
# the TFRecord pipeline, and to test loading trained model without tensors.
x_test = np.reshape(data.validation.images,
(data.validation.images.shape[0], 28, 28, 1))
y_test = data.validation.labels
x_test_inp = KL.Input(shape=(x_test.shape[1:]))
test_out = cnn_layers(x_test_inp, nclasses)
test_model = Model(inputs=x_test_inp, outputs=test_out)
test_model.load_weights('saved_wt.h5')
test_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
test_model.summary()
loss, acc = test_model.evaluate(x_test, to_categorical(y_test))
print('\nTest loss: {0}'.format(loss))
print('\nTest accuracy: {0}'.format(acc))
time.strftime('%Y-%m-%d_%H.%M.%S', time.gmtime())
os.mkdir('bb/models/' + run_id)
print('Loading data')
train_X = np.load('bb/data/sequences_train-' + str(TIMESTEPS) +
'steps.npy')[:, :, [0, 2, 4]]
val_X = np.load('bb/data/sequences_val-' + str(TIMESTEPS) +
'steps.npy')[:, :, [0, 2, 4]]
if len(train_X) % BATCH_SIZE != 0 or len(val_X) % BATCH_SIZE != 0:
raise ValueError(
'Data size incompatible with batch size (must be evenly divisible)')
num_inputs = len(train_X[0][0])
print('Number of inputs in loaded data: ' + str(num_inputs))
# Create graph structure.
input_placeholder = layers.Input(shape=[TIMESTEPS, num_inputs])
# Encoder.
encoded = layers.LSTM(64, return_sequences=True)(input_placeholder)
encoded = advanced_activations.ELU(alpha=.5)(encoded)
encoded = layers.LSTM(40)(encoded)
encoded = advanced_activations.ELU(alpha=.5)(encoded)
encoded = layers.Dense(3)(encoded)
encoded = advanced_activations.ELU(alpha=.5)(encoded)
encoded = layers.BatchNormalization(name='embedding')(encoded)
# Decoder.
decoded = layers.Dense(8)(encoded)
decoded = advanced_activations.ELU(alpha=.5)(decoded)
decoded = layers.Dense(16)(decoded)
decoded = advanced_activations.ELU(alpha=.5)(decoded)
decoded = layers.Dense(TIMESTEPS * num_inputs, activation='sigmoid')(decoded)
decoded = layers.Reshape([TIMESTEPS, num_inputs])(decoded)
def conv_dec(nb_features,
input_shape,
nb_levels,
conv_size,
nb_labels,
name=None,
prefix=None,
feat_mult=1,
pool_size=2,
use_skip_connections=False,
padding='same',
dilation_rate_mult=1,
activation='elu',
use_residuals=False,
final_pred_activation='softmax',
nb_conv_per_level=2,
layer_nb_feats=None,
batch_norm=None,
conv_dropout=0,
input_model=None):
"""
Fully Convolutional Decoder
Parameters:
...
use_skip_connections (bool): if true, turns an Enc-Dec to a U-Net.
If true, input_tensor and tensors are required.
It assumes a particular naming of layers. conv_enc...
"""
# naming
model_name = name
if prefix is None:
prefix = model_name
# if using skip connections, make sure need to use them.
if use_skip_connections:
assert input_model is not None, "is using skip connections, tensors dictionary is required"
# first layer: input
input_name = '%s_input' % prefix
if input_model is None:
input_tensor = KL.Input(shape=input_shape, name=input_name)
last_tensor = input_tensor
else:
input_tensor = input_model.input
last_tensor = input_model.output
input_shape = last_tensor.shape.as_list()[1:]
# vol size info
ndims = len(input_shape) - 1
input_shape = tuple(input_shape)
if isinstance(pool_size, int):
pool_size = (pool_size, ) * ndims
# prepare layers
convL = getattr(KL, 'Conv%dD' % ndims)
conv_kwargs = {'padding': padding, 'activation': activation}
upsample = getattr(KL, 'UpSampling%dD' % ndims)
# up arm:
# nb_levels - 1 layers of Deconvolution3D
# (approx via up + conv + ReLu) + merge + conv + ReLu + conv + ReLu
lfidx = 0
for level in range(nb_levels - 1):
nb_lvl_feats = np.round(nb_features *
feat_mult**(nb_levels - 2 - level)).astype(int)
conv_kwargs['dilation_rate'] = dilation_rate_mult**(nb_levels - 2 -
level)
# upsample matching the max pooling layers size
name = '%s_up_%d' % (prefix, nb_levels + level)
last_tensor = upsample(size=pool_size, name=name)(last_tensor)
up_tensor = last_tensor
# merge layers combining previous layer
# TODO: add Cropping3D or Cropping2D if 'valid' padding
if use_skip_connections:
conv_name = '%s_conv_downarm_%d_%d' % (
prefix, nb_levels - 2 - level, nb_conv_per_level - 1)
cat_tensor = input_model.get_layer(conv_name).output
name = '%s_merge_%d' % (prefix, nb_levels + level)
last_tensor = KL.concatenate([cat_tensor, last_tensor],
axis=ndims + 1,
name=name)
# convolution layers
for conv in range(nb_conv_per_level):
if layer_nb_feats is not None:
nb_lvl_feats = layer_nb_feats[lfidx]
lfidx += 1
name = '%s_conv_uparm_%d_%d' % (prefix, nb_levels + level, conv)
if conv < (nb_conv_per_level - 1) or (not use_residuals):
last_tensor = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(last_tensor)
else:
last_tensor = convL(nb_lvl_feats,
conv_size,
padding=padding,
name=name)(last_tensor)
if conv_dropout > 0:
name = '%s_dropout_uparm_%d_%d' % (prefix, level, conv)
noise_shape = [None, *[1] * ndims, nb_lvl_feats]
last_tensor = KL.Dropout(conv_dropout,
noise_shape=noise_shape)(last_tensor)
# residual block
if use_residuals:
# the "add" layer is the original input
# However, it may not have the right number of features to be added
add_layer = up_tensor
nb_feats_in = add_layer.get_shape()[-1]
nb_feats_out = last_tensor.get_shape()[-1]
if nb_feats_in > 1 and nb_feats_out > 1 and (nb_feats_in !=
nb_feats_out):
name = '%s_expand_up_merge_%d' % (prefix, level)
add_layer = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(add_layer)
if conv_dropout > 0:
name = '%s_dropout_up_merge_%d_%d' % (prefix, level, conv)
noise_shape = [None, *[1] * ndims, nb_lvl_feats]
last_tensor = KL.Dropout(
conv_dropout, noise_shape=noise_shape)(last_tensor)
name = '%s_res_up_merge_%d' % (prefix, level)
last_tensor = KL.add([last_tensor, add_layer], name=name)
name = '%s_res_up_merge_act_%d' % (prefix, level)
last_tensor = KL.Activation(activation, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_bn_up_%d' % (prefix, level)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# Compute likelyhood prediction (no activation yet)
name = '%s_likelihood' % prefix
last_tensor = convL(nb_labels, 1, activation=None, name=name)(last_tensor)
like_tensor = last_tensor
# output prediction layer
# we use a softmax to compute P(L_x|I) where x is each location
if final_pred_activation == 'softmax':
print("using final_pred_activation %s for %s" %
(final_pred_activation, model_name))
name = '%s_prediction' % prefix
softmax_lambda_fcn = lambda x: keras.activations.softmax(
x, axis=ndims + 1)
pred_tensor = KL.Lambda(softmax_lambda_fcn, name=name)(last_tensor)
# otherwise create a layer that does nothing.
else:
name = '%s_prediction' % prefix
pred_tensor = KL.Activation('linear', name=name)(like_tensor)
# create the model and retun
model = Model(inputs=input_tensor, outputs=pred_tensor, name=model_name)
return model
def CapsNet_WithDecoder(input_shape, n_class, kernel, primary_channel,
primary_veclen, digit_veclen, dropout, routings,
decoderParm):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
if kernel > 19:
conv_padding = 'same'
print('kernel size big, padding to same')
else:
conv_padding = 'valid'
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=kernel,
strides=1,
padding=conv_padding,
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=primary_veclen,
n_channels=primary_channel,
kernel_size=kernel,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=digit_veclen,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
NumDecoderLayer, DecoderLayerUnit = decoderParm
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(DecoderLayerUnit[0],
activation='relu',
input_dim=digit_veclen * n_class)) #16*n_class))
for i in range(NumDecoderLayer - 2):
decoder.add(layers.Dense(DecoderLayerUnit[i + 1], activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, digit_veclen))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def VGG19_dense(input_shape=None, classes=4):
img_input = layers.Input(shape=input_shape)
# Block 1
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer="he_normal")(img_input)
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
x = layers.Conv2D(32, (3, 3), activation='relu', name="feature")(x)
x = layers.Conv2D(32, (1, 1), activation='relu')(x)
x = layers.Conv2D(classes, (1, 1), activation='softmax')(x)
x = layers.AveragePooling2D((9, 9))(x)
model = models.Model(img_input, x, name='vgg19')
model.summary()
return model
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size,), name='states')
actions = layers.Input(shape=(self.action_size,), name='actions')
# Parameters for critic network
ki = initializers.glorot_uniform();
kr = None
if self.model_params["use_l2"] > 0:
l2 = self.model_params["use_l2"]
kr = regularizers.l2(l2)
dropout_rate = self.model_params["dropout_rate"]
use_bias = True
use_bn = self.model_params["use_bn"]
if use_bn:
use_bias=False
act_fn = self.model_params["act_fn"]
n1 = self.model_params["layer1_n"]
n2 = self.model_params["layer2_n"]
# The critic network
s_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='s_fc1')(states)
if use_bn:
s_fc1 = layers.BatchNormalization()(s_fc1)
if (dropout_rate > 0):
s_fc1 = layers.Dropout(dropout_rate)(s_fc1)
s_fc1 = layers.Activation(act_fn)(s_fc1)
a_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='a_fc1')(actions)
if use_bn:
a_fc1 = layers.BatchNormalization()(a_fc1)
if (dropout_rate > 0):
a_fc1 = layers.Dropout(dropout_rate)(a_fc1)
a_fc1 = layers.Activation(act_fn)(a_fc1)
# Combine state and action pathways
net = layers.Add()([s_fc1, a_fc1])
net = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(net)
if use_bn:
net = layers.BatchNormalization()(net)
if (dropout_rate > 0):
net = layers.Dropout(dropout_rate)(net)
net = layers.Activation(act_fn)(net)
# Add final output layer to prduce action values (Q values)
ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3)
Q_values = layers.Dense(units=1, kernel_initializer=ki, kernel_regularizer=kr,
activation='linear', name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(lr=0.001)
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def adversarial_training(data_dir, generator_model_path,
discriminator_model_path):
"""
Adversarial training of the generator network Gθ and discriminator network Dφ.
"""
#
# define model input and output tensors
#
generator_input_tensor = layers.Input(shape=(rand_dim, ))
generated_image_tensor = generator_network(generator_input_tensor)
generated_or_real_image_tensor = layers.Input(shape=(img_height, img_width,
img_channels))
discriminator_output = discriminator_network(
generated_or_real_image_tensor)
combined_output = discriminator_network(
generator_network(generator_input_tensor))
#
# define models
#
generator_model = models.Model(input=generator_input_tensor,
output=generated_image_tensor,
name='generator')
discriminator_model = models.Model(input=generated_or_real_image_tensor,
output=discriminator_output,
name='discriminator')
combined_model = models.Model(input=generator_input_tensor,
output=combined_output,
name='combined')
#
# compile models
#
adam = optimizers.Adam(
lr=0.0002, beta_1=0.5,
beta_2=0.999) # as described in appendix A of DeepMind's AC-GAN paper
generator_model.compile(optimizer=adam, loss='binary_crossentropy')
discriminator_model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['accuracy'])
discriminator_model.trainable = False
combined_model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['accuracy'])
print(generator_model.summary())
print(discriminator_model.summary())
#
# data generators
#
data_generator = image.ImageDataGenerator(
preprocessing_function=applications.xception.preprocess_input,
dim_ordering='tf')
flow_from_directory_params = {
'target_size': (img_height, img_width),
'color_mode': 'grayscale' if img_channels == 1 else 'rgb',
'class_mode': None,
'batch_size': batch_size
}
real_image_generator = data_generator.flow_from_directory(
directory=data_dir, **flow_from_directory_params)
def get_image_batch():
img_batch = real_image_generator.next()
# keras generators may generate an incomplete batch for the last batch
if len(img_batch) != batch_size:
img_batch = real_image_generator.next()
assert img_batch.shape == (batch_size, img_height, img_width,
img_channels), img_batch.shape
return img_batch
# the target labels for the binary cross-entropy loss layer are 0 for every yj (real) and 1 for every xi (generated)
y_real = np.array([0] * batch_size)
y_generated = np.array([1] * batch_size)
combined_loss = np.zeros(shape=len(combined_model.metrics_names))
disc_loss_real = np.zeros(shape=len(discriminator_model.metrics_names))
disc_loss_generated = np.zeros(
shape=len(discriminator_model.metrics_names))
if generator_model_path:
generator_model.load_weights(generator_model_path, by_name=True)
if discriminator_model_path:
discriminator_model.load_weights(discriminator_model_path,
by_name=True)
for i in range(nb_steps):
print('Step: {} of {}.'.format(i, nb_steps))
# train the discriminator
for _ in range(k_d):
generator_input = np.random.normal(size=(batch_size, rand_dim))
# sample a mini-batch of real images
real_image_batch = get_image_batch()
# generate a batch of images with the current generator
generated_image_batch = generator_model.predict(generator_input)
# update φ by taking an SGD step on mini-batch loss LD(φ)
disc_loss_real = np.add(
discriminator_model.train_on_batch(real_image_batch, y_real),
disc_loss_real)
disc_loss_generated = np.add(
discriminator_model.train_on_batch(generated_image_batch,
y_generated),
disc_loss_generated)
# train the generator
for _ in range(k_g * 2):
generator_input = np.random.normal(size=(batch_size, rand_dim))
# update θ by taking an SGD step on mini-batch loss LR(θ)
combined_loss = np.add(
combined_model.train_on_batch(generator_input, y_real),
combined_loss)
if not i % log_interval and i != 0:
# plot batch of generated images w/ current generator
figure_name = 'generated_image_batch_step_{}.png'.format(i)
print('Saving batch of generated images at adversarial step: {}.'.
format(i))
plot_images.plot_batch(
generator_model.predict(
np.random.normal(size=(batch_size, rand_dim))),
get_image_batch(), os.path.join(cache_dir, figure_name))
# log loss summary
print('Generator model loss: {}.'.format(combined_loss /
(log_interval * k_g * 2)))
print('Discriminator model loss real: {}.'.format(
disc_loss_real / (log_interval * k_d * 2)))
print('Discriminator model loss generated: {}.'.format(
disc_loss_generated / (log_interval * k_d * 2)))
combined_loss = np.zeros(shape=len(combined_model.metrics_names))
disc_loss_real = np.zeros(
shape=len(discriminator_model.metrics_names))
disc_loss_generated = np.zeros(
shape=len(discriminator_model.metrics_names))
# save model checkpoints
model_checkpoint_base_name = os.path.join(
cache_dir, '{}_model_weights_step_{}.h5')
generator_model.save_weights(
model_checkpoint_base_name.format('generator', i))
discriminator_model.save_weights(
model_checkpoint_base_name.format('discriminator', i))
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size,), name='states')
# Parameters for actor network
ki = initializers.glorot_uniform();
kr = None
if self.model_params["use_l2"] > 0:
l2 = self.model_params["use_l2"]
kr = regularizers.l2(l2)
dropout_rate = self.model_params["dropout_rate"]
use_bias = True
use_bn = self.model_params["use_bn"]
if use_bn:
use_bias=False
act_fn = self.model_params["act_fn"]
n1 = self.model_params["layer1_n"]
n2 = self.model_params["layer2_n"]
# the actor network
fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc1')(states)
if use_bn:
fc1 = layers.BatchNormalization()(fc1)
if (dropout_rate > 0):
fc1 = layers.Dropout(dropout_rate)(fc1)
fc1 = layers.Activation(act_fn)(fc1)
fc2 = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(fc1)
if use_bn:
fc2 = layers.BatchNormalization()(fc2)
if (dropout_rate > 0):
fc2 = layers.Dropout(dropout_rate)(fc2)
fc2 = layers.Activation(act_fn)(fc2)
# Add final output layer
ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3)
raw_actions = layers.Dense(units=self.action_size, activation='tanh',
kernel_initializer=ki, kernel_regularizer=kr,
use_bias=False, name='raw_actions')(fc2)
# Scale [-1, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x: (0.5*(x + 1.0)*self.action_range + self.action_low),
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-action_gradients * actions)
if (kr is not None):
loss = loss + l2*K.sum(K.square(self.model.get_layer('fc1').get_weights()[0])) \
+ l2*K.sum(K.square(self.model.get_layer('fc2').get_weights()[0])) \
+ l2*K.sum(K.square(self.model.get_layer('raw_actions').get_weights()[0]))
# Define optimizer and training function
optimizer = optimizers.Adam(lr=0.0001)
updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients, K.learning_phase()],
outputs=[],
updates=updates_op)
def densenet(channels,
init_block_channels,
dropout_rate=0.0,
in_channels=3,
in_size=(224, 224),
classes=1000):
"""
DenseNet model from 'Densely Connected Convolutional Networks,' https://arxiv.org/abs/1608.06993.
Parameters:
----------
channels : list of list of int
Number of output channels for each unit.
init_block_channels : int
Number of output channels for the initial unit.
dropout_rate : float, default 0.0
Parameter of Dropout layer. Faction of the input units to drop.
in_channels : int, default 3
Number of input channels.
in_size : tuple of two ints, default (224, 224)
Spatial size of the expected input image.
classes : int, default 1000
Number of classification classes.
"""
input_shape = (in_channels, 224, 224) if is_channels_first() else (224, 224, in_channels)
input = nn.Input(shape=input_shape)
x = preres_init_block(
x=input,
in_channels=in_channels,
out_channels=init_block_channels,
name="features/init_block")
in_channels = init_block_channels
for i, channels_per_stage in enumerate(channels):
if i != 0:
x = transition_block(
x=x,
in_channels=in_channels,
out_channels=(in_channels // 2),
name="features/stage{}/trans{}".format(i + 1, i + 1))
in_channels = in_channels // 2
for j, out_channels in enumerate(channels_per_stage):
x = dense_unit(
x=x,
in_channels=in_channels,
out_channels=out_channels,
dropout_rate=dropout_rate,
name="features/stage{}/unit{}".format(i + 1, j + 1))
in_channels = out_channels
x = preres_activation(
x=x,
name="features/post_activ")
x = nn.AvgPool2D(
pool_size=7,
strides=1,
name="features/final_pool")(x)
# x = nn.Flatten()(x)
x = flatten(x)
x = nn.Dense(
units=classes,
input_dim=in_channels,
name="output")(x)
model = Model(inputs=input, outputs=x)
model.in_size = in_size
model.classes = classes
return model
def VGG19(input_shape=None, classes=[3, 5], use_soft=True):
img_input = layers.Input(shape=input_shape)
# Block 1
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer="he_normal")(img_input)
x = layers.Conv2D(2, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(4, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(8, (3, 3),
activation='relu',
padding='same',
name='block3_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block4_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer="he_normal")(x)
x = layers.Conv2D(16, (3, 3),
activation='relu',
padding='same',
name='block5_conv4',
kernel_initializer="he_normal")(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# Classification block
x = layers.Flatten(name='flatten')(x)
x1 = layers.Dense(512, activation='relu', name='fc1')(x)
x1 = layers.Dropout(0.5, name="dropout_1")(x1)
x1 = layers.Dense(128, activation='relu', name='fc2')(x1)
#x=layers.Dropout(0.8)(x)
x2 = layers.Dense(512, activation='relu', name='fc1_branch')(x)
x2 = layers.Dropout(0.5, name="dropout_1_branch")(x2)
x2 = layers.Dense(128, activation='relu', name='fc2_branch')(x2)
if use_soft:
x1 = Dense(classes[0], activation="softmax", name='predictions')(x1)
x2 = Dense(classes[1], activation="softmax",
name='predictions_branch')(x2)
else:
x1 = Dense(classes[0], activation="linear", name="Z_4")(x)
model = models.Model(img_input, [x1, x2], name='vgg19')
return model
def single_ae(
enc_size,
input_shape,
name='single_ae',
prefix=None,
ae_type='dense', # 'dense', or 'conv'
conv_size=None,
input_model=None,
enc_lambda_layers=None,
batch_norm=True,
padding='same',
activation=None,
include_mu_shift_layer=False,
do_vae=False):
"""
single-layer Autoencoder (i.e. input - encoding - output)
"""
# naming
model_name = name
if prefix is None:
prefix = model_name
if enc_lambda_layers is None:
enc_lambda_layers = []
# prepare input
input_name = '%s_input' % prefix
if input_model is None:
assert input_shape is not None, 'input_shape of input_model is necessary'
input_tensor = KL.Input(shape=input_shape, name=input_name)
last_tensor = input_tensor
else:
input_tensor = input_model.input
last_tensor = input_model.output
input_shape = last_tensor.shape.as_list()[1:]
input_nb_feats = last_tensor.shape.as_list()[-1]
# prepare conv type based on input
if ae_type == 'conv':
ndims = len(input_shape) - 1
convL = getattr(KL, 'Conv%dD' % ndims)
assert conv_size is not None, 'with conv ae, need conv_size'
conv_kwargs = {'padding': padding, 'activation': activation}
# if want to go through a dense layer in the middle of the U, need to:
# - flatten last layer if not flat
# - do dense encoding and decoding
# - unflatten (rehsape spatially) at end
if ae_type == 'dense' and len(input_shape) > 1:
name = '%s_ae_%s_down_flat' % (prefix, ae_type)
last_tensor = KL.Flatten(name=name)(last_tensor)
# recall this layer
pre_enc_layer = last_tensor
# encoding layer
if ae_type == 'dense':
assert len(
enc_size) == 1, "enc_size should be of length 1 for dense layer"
enc_size_str = ''.join(['%d_' % d for d in enc_size])[:-1]
name = '%s_ae_mu_enc_dense_%s' % (prefix, enc_size_str)
last_tensor = KL.Dense(enc_size[0], name=name)(pre_enc_layer)
else: # convolution
# convolve then resize. enc_size should be [nb_dim1, nb_dim2, ..., nb_feats]
assert len(enc_size) == len(input_shape), \
"encoding size does not match input shape %d %d" % (len(enc_size), len(input_shape))
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
# assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing -- need to check out interpn!"
name = '%s_ae_mu_enc_conv' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
name = '%s_ae_mu_enc' % (prefix)
zf = [
enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f]
for f in range(len(enc_size) - 1)
]
last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor)
# resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1])
# last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
elif enc_size[
-1] is None: # convolutional, but won't tell us bottleneck
name = '%s_ae_mu_enc' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer)
else:
name = '%s_ae_mu_enc' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
if include_mu_shift_layer:
# shift
name = '%s_ae_mu_shift' % (prefix)
last_tensor = layers.LocalBias(name=name)(last_tensor)
# encoding clean-up layers
for layer_fcn in enc_lambda_layers:
lambda_name = layer_fcn.__name__
name = '%s_ae_mu_%s' % (prefix, lambda_name)
last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_ae_mu_bn' % (prefix)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# have a simple layer that does nothing to have a clear name before sampling
name = '%s_ae_mu' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor)
# if doing variational AE, will need the sigma layer as well.
if do_vae:
mu_tensor = last_tensor
# encoding layer
if ae_type == 'dense':
name = '%s_ae_sigma_enc_dense_%s' % (prefix, enc_size_str)
last_tensor = KL.Dense(
enc_size[0],
name=name,
# kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=1e-5),
# bias_initializer=keras.initializers.RandomNormal(mean=-5.0, stddev=1e-5)
)(pre_enc_layer)
else:
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
# assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing..."
name = '%s_ae_sigma_enc_conv' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
name = '%s_ae_sigma_enc' % (prefix)
zf = [
enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f]
for f in range(len(enc_size) - 1)
]
last_tensor = layers.Resize(zoom_factor=zf,
name=name)(last_tensor)
# resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1])
# last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
elif enc_size[
-1] is None: # convolutional, but won't tell us bottleneck
name = '%s_ae_sigma_enc' % (prefix)
last_tensor = convL(pre_enc_layer.shape.as_list()[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
# cannot use lambda, then mu and sigma will be same layer.
# last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer)
else:
name = '%s_ae_sigma_enc' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
# encoding clean-up layers
for layer_fcn in enc_lambda_layers:
lambda_name = layer_fcn.__name__
name = '%s_ae_sigma_%s' % (prefix, lambda_name)
last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_ae_sigma_bn' % (prefix)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# have a simple layer that does nothing to have a clear name before sampling
name = '%s_ae_sigma' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor)
logvar_tensor = last_tensor
# VAE sampling
name = '%s_ae_sample' % (prefix)
last_tensor = layers.SampleNormalLogVar(name=name)(
[mu_tensor, logvar_tensor])
if include_mu_shift_layer:
# shift
name = '%s_ae_sample_shift' % (prefix)
last_tensor = layers.LocalBias(name=name)(last_tensor)
# decoding layer
if ae_type == 'dense':
name = '%s_ae_%s_dec_flat_%s' % (prefix, ae_type, enc_size_str)
last_tensor = KL.Dense(np.prod(input_shape), name=name)(last_tensor)
# unflatten if dense method
if len(input_shape) > 1:
name = '%s_ae_%s_dec' % (prefix, ae_type)
last_tensor = KL.Reshape(input_shape, name=name)(last_tensor)
else:
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
name = '%s_ae_mu_dec' % (prefix)
zf = [
input_shape[:-1][f] / enc_size[:-1][f]
for f in range(len(enc_size) - 1)
]
last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor)
# resize_fn = lambda x: tf.image.resize_bilinear(x, input_shape[:-1])
# last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
name = '%s_ae_%s_dec' % (prefix, ae_type)
last_tensor = convL(input_nb_feats,
conv_size,
name=name,
**conv_kwargs)(last_tensor)
if batch_norm is not None:
name = '%s_bn_ae_%s_dec' % (prefix, ae_type)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# create the model and retun
model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name)
return model
def build_model():
from keras import models, layers
# Build U-Net model
def upsample_conv(filters, kernel_size, strides, padding):
return layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding)
def upsample_simple(filters, kernel_size, strides, padding):
return layers.UpSampling2D(strides)
if UPSAMPLE_MODE=='DECONV':
upsample=upsample_conv
else:
upsample=upsample_simple
input_img = layers.Input(t_x.shape[1:], name = 'RGB_Input')
pp_in_layer = input_img
if NET_SCALING is not None:
pp_in_layer = layers.AvgPool2D(NET_SCALING)(pp_in_layer)
pp_in_layer = layers.GaussianNoise(GAUSSIAN_NOISE)(pp_in_layer)
pp_in_layer = layers.BatchNormalization()(pp_in_layer)
c1 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (pp_in_layer)
c1 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (c1)
p1 = layers.MaxPooling2D((2, 2)) (c1)
c2 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (p1)
c2 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (c2)
p2 = layers.MaxPooling2D((2, 2)) (c2)
c3 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (p2)
c3 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (c3)
p3 = layers.MaxPooling2D((2, 2)) (c3)
c4 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (p3)
c4 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (c4)
p4 = layers.MaxPooling2D(pool_size=(2, 2)) (c4)
c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same') (p4)
c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same') (c5)
u6 = upsample(64, (2, 2), strides=(2, 2), padding='same') (c5)
u6 = layers.concatenate([u6, c4])
c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (u6)
c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (c6)
u7 = upsample(32, (2, 2), strides=(2, 2), padding='same') (c6)
u7 = layers.concatenate([u7, c3])
c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (u7)
c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (c7)
u8 = upsample(16, (2, 2), strides=(2, 2), padding='same') (c7)
u8 = layers.concatenate([u8, c2])
c8 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (u8)
c8 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (c8)
u9 = upsample(8, (2, 2), strides=(2, 2), padding='same') (c8)
u9 = layers.concatenate([u9, c1], axis=3)
c9 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (u9)
c9 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (c9)
d = layers.Conv2D(1, (1, 1), activation='sigmoid') (c9)
# d = layers.Cropping2D((EDGE_CROP, EDGE_CROP))(d)
# d = layers.ZeroPadding2D((EDGE_CROP, EDGE_CROP))(d)
if NET_SCALING is not None:
d = layers.UpSampling2D(NET_SCALING)(d)
seg_model = models.Model(inputs=[input_img], outputs=[d])
seg_model.summary()
