def MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000):
"""Instantiates the MobileNet architecture.
Arguments:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or (3, 224, 224) (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(200, 200, 3)` would be one valid value.
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: depth multiplier for depthwise convolution
(also called the resolution multiplier)
dropout: dropout rate
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
# Determine proper input shape and default size.
if input_shape is None:
default_size = 224
else:
if K.image_data_format() == 'channels_first':
rows = input_shape[1]
cols = input_shape[2]
else:
rows = input_shape[0]
cols = input_shape[1]
if rows == cols and rows in [128, 160, 192, 224]:
default_size = rows
else:
default_size = 224
input_shape = _obtain_input_shape(input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if weights == 'imagenet':
if depth_multiplier != 1:
raise ValueError('If imagenet weights are being loaded, '
'depth multiplier must be 1')
if alpha not in [0.25, 0.50, 0.75, 1.0]:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of'
'`0.25`, `0.50`, `0.75` or `1.0` only.')
if rows != cols or rows not in [128, 160, 192, 224]:
if rows is None:
rows = 224
logging.warning(
'MobileNet shape is undefined.'
' Weights for input shape (224, 224) will be loaded.')
else:
raise ValueError(
'If imagenet weights are being loaded, '
'input must have a static square shape (one of '
'(128, 128), (160, 160), (192, 192), or (224, 224)).'
' Input shape provided = %s' % (input_shape, ))
if K.image_data_format() != 'channels_last':
logging.warning(
'The MobileNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = _conv_block(img_input, 32, alpha, strides=(2, 2))
x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1)
x = _depthwise_conv_block(x,
128,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=2)
x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3)
x = _depthwise_conv_block(x,
256,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=4)
x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5)
x = _depthwise_conv_block(x,
512,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=6)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11)
x = _depthwise_conv_block(x,
1024,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=12)
x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13)
if include_top:
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
x = GlobalAveragePooling2D()(x)
x = Reshape(shape, name='reshape_1')(x)
#x = Dropout(dropout, name='dropout')(x)
x = Conv2D(classes, (1, 1), padding='same', name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = Reshape((classes, ), name='reshape_2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='mobilenet_%0.2f_%s' % (alpha, rows))
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
raise ValueError('Weights for "channels_first" format '
'are not available.')
if alpha == 1.0:
alpha_text = '1_0'
elif alpha == 0.75:
alpha_text = '7_5'
elif alpha == 0.50:
alpha_text = '5_0'
else:
alpha_text = '2_5'
if include_top:
model_name = 'mobilenet_%s_%d_tf.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
else:
model_name = 'mobilenet_%s_%d_tf_no_top.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
def network_model(input_shape, input_name, num_classes):
X_input = Input(shape=input_shape, name=input_name)
# Stage 1
X = Conv2D(64, (7, 7),
strides=(1, 1),
padding='same',
name='conv1',
kernel_initializer=glorot_uniform(seed=0))(X_input)
X = BatchNormalization(axis=3, name='bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3), strides=(1, 1), padding='same')(X)
# Stage 2
X = convolutional_block(X,
f=3,
filters=[64, 64, 256],
stage=2,
block='a',
s=1)
X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')
X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')
# Stage 3
X = convolutional_block(X,
f=3,
filters=[128, 128, 512],
stage=3,
block='a',
s=2)
X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')
# Stage 4
X = convolutional_block(X,
f=3,
filters=[256, 256, 1024],
stage=4,
block='a',
s=2)
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')
# Stage 5
X = convolutional_block(X,
f=3,
filters=[512, 512, 2048],
stage=5,
block='a',
s=2)
X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')
X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')
# Average pooling
X = AveragePooling2D(pool_size=(7, 7), strides=(1, 1), name='avg_pool')(X)
# output layer
X = Flatten()(X)
X = Dense(num_classes,
activation='softmax',
name='fc' + str(num_classes),
kernel_initializer=glorot_uniform(seed=0))(X)
model = Model(inputs=X_input, outputs=X, name='Resnet50')
return model
def DSIN(
dnn_feature_columns,
sess_feature_list,
sess_max_count=5,
bias_encoding=False,
att_embedding_size=1,
att_head_num=8,
dnn_hidden_units=(256, 128, 64),
dnn_activation='sigmoid',
dnn_dropout=0,
dnn_use_bn=False,
l2_reg_dnn=0,
l2_reg_embedding=1e-6,
seed=1024,
task='binary',
):
"""Instantiates the Deep Session Interest Network architecture.
:param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
:param sess_feature_list: list,to indicate sequence sparse field
:param sess_max_count: positive int, to indicate the max number of sessions
:param sess_len_max: positive int, to indicate the max length of each session
:param bias_encoding: bool. Whether use bias encoding or postional encoding
:param att_embedding_size: positive int, the embedding size of each attention head
:param att_head_num: positive int, the number of attention head
:param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param dnn_activation: Activation function to use in deep net
:param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
:param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in deep net
:param l2_reg_dnn: float. L2 regularizer strength applied to DNN
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param seed: integer ,to use as random seed.
:param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss
:return: A Keras model instance.
"""
hist_emb_size = sum(
map(
lambda fc: fc.embedding_dim,
filter(lambda fc: fc.name in sess_feature_list,
dnn_feature_columns)))
if (att_embedding_size * att_head_num != hist_emb_size):
raise ValueError(
"hist_emb_size must equal to att_embedding_size * att_head_num ,got %d != %d *%d"
% (hist_emb_size, att_embedding_size, att_head_num))
features = build_input_features(dnn_feature_columns)
sparse_feature_columns = list(
filter(lambda x: isinstance(x, SparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
dense_feature_columns = list(
filter(lambda x: isinstance(x, DenseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
varlen_sparse_feature_columns = list(
filter(lambda x: isinstance(x, VarLenSparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
sparse_varlen_feature_columns = []
history_fc_names = list(map(lambda x: "sess" + x, sess_feature_list))
for fc in varlen_sparse_feature_columns:
feature_name = fc.name
if feature_name in history_fc_names:
continue
else:
sparse_varlen_feature_columns.append(fc)
inputs_list = list(features.values())
user_behavior_input_dict = {}
for idx in range(sess_max_count):
sess_input = OrderedDict()
for i, feat in enumerate(sess_feature_list):
sess_input[feat] = features["sess_" + str(idx) + "_" + feat]
user_behavior_input_dict["sess_" + str(idx)] = sess_input
user_sess_length = Input(shape=(1, ), name='sess_length')
embedding_dict = {
feat.embedding_name:
Embedding(feat.vocabulary_size,
feat.embedding_dim,
embeddings_initializer=feat.embeddings_initializer,
embeddings_regularizer=l2(l2_reg_embedding),
name='sparse_emb_' + str(i) + '-' + feat.name,
mask_zero=(feat.name in sess_feature_list))
for i, feat in enumerate(sparse_feature_columns)
}
query_emb_list = embedding_lookup(embedding_dict,
features,
sparse_feature_columns,
sess_feature_list,
sess_feature_list,
to_list=True)
dnn_input_emb_list = embedding_lookup(embedding_dict,
features,
sparse_feature_columns,
mask_feat_list=sess_feature_list,
to_list=True)
dense_value_list = get_dense_input(features, dense_feature_columns)
query_emb = concat_func(query_emb_list, mask=True)
dnn_input_emb = Flatten()(concat_func(dnn_input_emb_list))
tr_input = sess_interest_division(embedding_dict,
user_behavior_input_dict,
sparse_feature_columns,
sess_feature_list,
sess_max_count,
bias_encoding=bias_encoding)
Self_Attention = Transformer(att_embedding_size,
att_head_num,
dropout_rate=0,
use_layer_norm=False,
use_positional_encoding=(not bias_encoding),
seed=seed,
supports_masking=True,
blinding=True)
sess_fea = sess_interest_extractor(tr_input, sess_max_count,
Self_Attention)
interest_attention_layer = AttentionSequencePoolingLayer(
att_hidden_units=(64, 16),
weight_normalization=True,
supports_masking=False)([query_emb, sess_fea, user_sess_length])
lstm_outputs = BiLSTM(
hist_emb_size,
layers=2,
res_layers=0,
dropout_rate=0.2,
)(sess_fea)
lstm_attention_layer = AttentionSequencePoolingLayer(
att_hidden_units=(64, 16),
weight_normalization=True)([query_emb, lstm_outputs, user_sess_length])
dnn_input_emb = Concatenate()([
dnn_input_emb,
Flatten()(interest_attention_layer),
Flatten()(lstm_attention_layer)
])
dnn_input_emb = combined_dnn_input([dnn_input_emb], dense_value_list)
output = DNN(dnn_hidden_units,
dnn_activation,
l2_reg_dnn,
dnn_dropout,
dnn_use_bn,
seed=seed)(dnn_input_emb)
output = Dense(
1,
use_bias=False,
kernel_initializer=tf.keras.initializers.glorot_normal(seed))(output)
output = PredictionLayer(task)(output)
sess_input_list = []
for i in range(sess_max_count):
sess_name = "sess_" + str(i)
sess_input_list.extend(
get_inputs_list([user_behavior_input_dict[sess_name]]))
model = Model(inputs=inputs_list + [user_sess_length], outputs=output)
return model
def FCN(num_classes , input_height=224, input_width=224,vgg_weight_path=None):
img_input = Input(shape=(input_height,input_width, 3))
# Block 1
x = Conv2D(64, (3, 3), padding='same', name='block1_conv1')(img_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', name='block1_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Block 2
x = Conv2D(128, (3, 3), padding='same', name='block2_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', name='block2_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Block 3
x = Conv2D(256, (3, 3), padding='same', name='block3_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='block3_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='block3_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
block_3_out = MaxPooling2D()(x)
# Block 4
x = Conv2D(512, (3, 3), padding='same', name='block4_conv1')(block_3_out)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block4_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block4_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
block_4_out = MaxPooling2D()(x)
# Block 5
x = Conv2D(512, (3, 3), padding='same', name='block5_conv1')(block_4_out)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block5_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block5_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Load pretrained weights.
#if vgg_weight_path is not None:
#vgg16 = Model(img_input, x)
#vgg16.load_weights(vgg_weight_path, by_name=True)
# Convolutinalized fully connected layer.
x = Conv2D(4096, (7, 7), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(4096, (1, 1), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Classifying layers.
x = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(x)
x = BatchNormalization()(x)
block_3_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_3_out)
block_3_out = BatchNormalization()(block_3_out)
block_4_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_4_out)
block_4_out = BatchNormalization()(block_4_out)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x)
x = Add()([x, block_4_out])
x = Activation('relu')(x)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x)
x = Add()([x, block_3_out])
x = Activation('relu')(x)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 8, x.shape[2] * 8)))(x)
x = Activation('softmax')(x)
model = Model(img_input, x)
return model
def __init__(self, input_dim, dr_rate=0.2, opt_name='sgd', lr=0.001, initializer='he_uniform', batchnorm=False):
self.input_dim = input_dim
self.dr_rate = dr_rate
self.opt_name = opt_name
self.initializer = initializer
self.lr = lr
# layers = [1000, 1000, 500, 250, 125, 60, 30]
inputs = Input(shape=(self.input_dim,), name='inputs')
# inputs = Input(shape=(input_dim,))
#x = Lambda(lambda x: x, output_shape=(1000,))(inputs)
# attn_lin = Dense(1000, activation='relu', name='attn_lin')(inputs)
# attn_probs = Dense(1000, activation='softmax', name='attn_probs')(inputs)
# x = keras.layers.multiply( [attn_lin, attn_probs], name='attn')
# ------------------------------------------
# New attention layer (Rick, Austin)
"""
a = Dense(1000)(inputs)
a = BatchNormalization()(a)
a = Activation('relu')(a)
b = Attention(1000)(a)
x = keras.layers.multiply([b, a])
"""
# Old attention layer
a = Dense(1000, activation='relu')(inputs)
b = Dense(1000, activation='softmax')(inputs)
x = keras.layers.multiply( [a, b] )
x = Dense(1000)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(500)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(250)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(125)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(60)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(30)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
# ------------------------------------------
outputs = Dense(1, activation='relu')(x)
model = Model(inputs=inputs, outputs=outputs)
# if opt_name == 'sgd':
# opt = SGD(lr=1e-4, momentum=0.9)
# elif opt_name == 'adam':
# opt = Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
# else:
# opt = SGD(lr=1e-4, momentum=0.9) # for clr
opt = self.get_optimizer()
model.compile(loss='mean_squared_error', optimizer=opt, metrics=['mae']) # r2_krs
self.model = model
def create_tf_emotionsrecognision_resnet18(size, alpha):
input_tensor = Input(shape=(size, size, 1))
output_tensor = ResNet18SI(input_tensor, alpha, 0)
output_tensor = Dense(7)(output_tensor)
return Model(inputs=input_tensor, outputs=output_tensor)
def layer_test(layer_cls,
kwargs={},
input_shape=None,
input_dtype=None,
input_data=None,
expected_output=None,
expected_output_dtype=None,
fixed_batch_size=False,
supports_masking=False):
# generate input data
if input_data is None:
if not input_shape:
raise AssertionError()
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
input_mask = []
if all(isinstance(e, tuple) for e in input_data_shape):
input_data = []
for e in input_data_shape:
input_data.append(
(10 * np.random.random(e)).astype(input_dtype))
if supports_masking:
a = np.full(e[:2], False)
a[:, :e[1] // 2] = True
input_mask.append(a)
else:
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
if supports_masking:
a = np.full(input_data_shape[:2], False)
a[:, :input_data_shape[1] // 2] = True
print(a)
print(a.shape)
input_mask.append(a)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except Exception:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if isinstance(input_shape, list):
if fixed_batch_size:
x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [
Input(batch_shape=e[0:2], dtype=bool) for e in input_shape
]
else:
x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [Input(shape=(e[1], ), dtype=bool) for e in input_shape]
else:
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
if supports_masking:
mask = Input(batch_shape=input_shape[0:2], dtype=bool)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
if supports_masking:
mask = Input(shape=(input_shape[1], ), dtype=bool)
if supports_masking:
y = layer(Masking()(x), mask=mask)
else:
y = layer(x)
if not (K.dtype(y) == expected_output_dtype):
raise AssertionError()
# check with the functional API
if supports_masking:
model = Model([x, mask], y)
actual_output = model.predict([input_data, input_mask[0]])
else:
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
if not (expected_dim == actual_dim):
raise AssertionError("expected_shape", expected_output_shape,
"actual_shape", actual_output_shape)
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
def FPNet(backbone,
input_shape,
inputs=None,
norm_method='whole_image',
use_imagenet=False,
pooling=None,
required_channels=3,
n_classes=3,
name='fpnet',
**kwargs):
"""Creates a Feature Pyramid Network with a semantic segmentation head
Args:
backbone (str): A name of a supported backbone from [deepcell, resnet50]
input_shape (tuple): Shape of the input image
input (keras layer, optional): Defaults to None. Method to pass in preexisting layers
norm_method (str, optional): Defaults to 'whole_image'. Normalization method
weights (str, optional): Defaults to None. one of None (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling (str, optional): Defaults to None. optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int, optional): Defaults to 3. The required number of channels of the
backbone. 3 is the default for all current backbones.
n_classes (int, optional): Defaults to 3. The number of classes to be predicted
name (str, optional): Defaults to 'fpnet'. Name to use for the model.
Returns:
keras.models.Model: Feature pyramid network with a semantic
segmentation head as the output
"""
if inputs is None:
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be required_channels
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
# Get backbone outputs
backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
**model_kwargs)
# Construct feature pyramid network
pyramid_dict = __create_pyramid_features(backbone_dict)
levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(levels)
x = __create_semantic_head(pyramid_dict,
n_classes=n_classes,
input_target=inputs,
target_level=target_level)
return Model(inputs=inputs, outputs=x, name=name)
def captions_to_tokens(self, captions_listlist):
tokens = [
self.texts_to_sequences(captions_list)
for captions_list in captions_listlist
]
return tokens
#获取开始标记的整数标记；获取结束标记的整数标记；将训练集中的所有描述转换为整数标记序列，我们得到了一个列表中的列表。
tokenizer = TokenizerWrap(texts=captions_train_flat, num_words=num_words)
token_start = tokenizer.word_index[mark_start.strip()]
token_end = tokenizer.word_index[mark_end.strip()]
state_size = 512
embedding_size = 128
transfer_values_input = Input(shape=(transfer_values_size, ),
name='transfer_values_input')
decoder_transfer_map = Dense(state_size,
activation='tanh',
name='decoder_transfer_map')
decoder_input = Input(shape=(None, ), name='decoder_input')
decoder_embedding = Embedding(input_dim=num_words,
output_dim=embedding_size,
name='decoder_embedding')
#定义RNN的3个GRU层
decoder_gru1 = GRU(state_size, name='decoder_gru1', return_sequences=True)
decoder_gru2 = GRU(state_size, name='decoder_gru2', return_sequences=True)
decoder_gru3 = GRU(state_size, name='decoder_gru3', return_sequences=True)
decoder_dense = Dense(num_words, activation='linear', name='decoder_output')
def ColorModel(img_height=227, img_width=227):
print('Building network')
_ = ['TOP', 'BOTTOM']
image_input = Input(shape=(img_height, img_width, 3), name='image_input')
image_output = []
for i in range(2):
x = conv_block(tensor=image_input,
nfilters=48,
size=11,
stride=4,
bn=True,
padding='valid',
name=f'{_[i]}_block_1')
top, bottom = conv_block(tensor=x,
nfilters=64,
block_type='split',
bn=True,
pool=True,
name=f'{_[i]}_block_2')
top = conv_block(tensor=top,
nfilters=96,
size=3,
bn=True,
pool=False,
name=f'{_[i]}_block_3t')
bottom = conv_block(tensor=bottom,
nfilters=96,
size=3,
bn=True,
pool=False,
name=f'{_[i]}_block_3b')
x = concatenate([top, bottom],
axis=-1,
name=f'{_[i]}_block3_concatenate')
top, bottom = conv_block(tensor=x,
nfilters=96,
block_type='split',
pool=False,
name=f'{_[i]}_block_4')
top = conv_block(tensor=top,
nfilters=64,
size=3,
bn=True,
pool=True,
name=f'{_[i]}_block_5t')
bottom = conv_block(tensor=bottom,
nfilters=64,
size=3,
bn=True,
pool=True,
name=f'{_[i]}_block_5b')
image_output.append(top)
image_output.append(bottom)
image_output = concatenate(image_output, axis=-1, name='features_output')
image_output = Flatten(name='flatten')(image_output)
image_output = Dense(4096,
activation='softmax',
kernel_initializer='he_normal',
name='fc1')(image_output)
image_output = Dense(4096,
activation='softmax',
kernel_initializer='he_normal',
name='fc2')(image_output)
image_output = Dense(4096,
activation='softmax',
kernel_initializer='he_normal',
name='fc3')(image_output)
image_output = Dense(20,
activation='softmax',
kernel_initializer='he_normal',
name='color_output')(image_output)
print('Successful')
return Model(inputs=image_input, outputs=image_output, name='Color-Model')
def get_rnn_model(MAX_NB_WORDS, embedding_matrix_2):
"""
Creating the RNN model
:param MAX_NB_WORDS: maximum length of the seuquence
:param embedding_matrix_2: embedding matrix for the tokens
"""
# defining input shape of the data
inp = Input(shape=(50, ))
# the layers
# -------------------------------------------
# defining Embedding layer
x = Embedding(input_dim=MAX_NB_WORDS,
ouput_dim=300,
input_length=50,
weights=[embedding_matrix_2],
trainable=False)(inp)
# ----------------------------------------------------
# dropout layer
x = SpatialDropout1D(0.2)(x)
# -------------------------------------------------------------------------
# defining RRN part
# two successive bidirectional GRU layers followed by a 1D Conv layer improved the performance
""" after some trial and error this combination was found to be the best. dono really why :(
GRU chosen over LSTM since it is simpler and faster
"""
x = Bidirectional(GRU(100, return_sequences=True))(x)
x = Bidirectional(GRU(100, return_sequences=True))(x)
x = Conv1D(512,
kernel_size=1,
padding="valid",
kernel_initializer="he_uniform")(x)
# --------------------------------------------------------------------------
# defining two pooling layers average and maximum. so we reduce the dimensionality
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
# concatenating the two pooling layers
conc = concatenate([avg_pool, max_pool])
# -------------------------------------------------------------------------
# applying batch normalization to speed the weights learning
"""
standardizes the input values. mean = 0, std = 1
"""
conc = BatchNormalization()(conc)
# -------------------------------------------------------------------------
conc = LeakyReLU()(conc)
# -------------------------------------------------------------------------
# defining the ouput layer
outp = Dense(10, activation='softmax')(conc)
# --------------------------------------------------------------------
# defining the model
model = Model(inputs=inp, outputs=outp)
# if we want to load pre-trained weights
# .load_weights("/weights-improvement-01-0.69.hdf5")
# Compiling the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def SqueezeNet(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
model_path=""):
"""Instantiates the SqueezeNet architecture.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
input_shape = _obtain_input_shape(input_shape,
default_size=227,
min_size=48,
data_format=K.image_data_format(),
require_flatten=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = Convolution2D(64, (3, 3),
strides=(2, 2),
padding='valid',
name='conv1')(img_input)
x = Activation('relu', name='relu_conv1')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool1')(x)
x = fire_module(x, fire_id=2, squeeze=16, expand=64)
x = fire_module(x, fire_id=3, squeeze=16, expand=64)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool3')(x)
x = fire_module(x, fire_id=4, squeeze=32, expand=128)
x = fire_module(x, fire_id=5, squeeze=32, expand=128)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool5')(x)
x = fire_module(x, fire_id=6, squeeze=48, expand=192)
x = fire_module(x, fire_id=7, squeeze=48, expand=192)
x = fire_module(x, fire_id=8, squeeze=64, expand=256)
x = fire_module(x, fire_id=9, squeeze=64, expand=256)
if include_top:
# It's not obvious where to cut the network...
# Could do the 8th or 9th layer... some work recommends cutting earlier layers.
x = Dropout(0.5, name='drop9')(x)
x = Convolution2D(classes, (1, 1), padding='valid', name='conv10')(x)
x = Activation('relu', name='relu_conv10')(x)
x = GlobalAveragePooling2D()(x)
x = Activation('softmax', name='loss')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
elif pooling == None:
pass
else:
raise ValueError("Unknown argument for 'pooling'=" + pooling)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = img_input
model = Model(inputs, x, name='squeezenet')
# load weights
if weights == 'imagenet':
weights_path = model_path
model.load_weights(weights_path)
return model
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size *
4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
if len(out_index) == 0: out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(
count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if args.plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
def valid_generator(valid_batch_size):
while True:
ids = list(range(valid_df.shape[0]))
for start in range(0, len(ids), valid_batch_size):
x_batch = []
y_batch = []
end = min(start + valid_batch_size, len(ids))
i_valid_batch = ids[start:end]
for i in i_valid_batch:
x_batch.append(process_wav_file(valid_df.wav_file.values[i]))
y_batch.append(valid_df.label_id.values[i])
x_batch = np.array(x_batch)
y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS))
yield x_batch, y_batch
x = Input(shape = (257,98,2))
x = BatchNormalization()(x)
num_filters = [64, 64, 128, 128, 256]
num_convs = [1, 2, 1, 1, 1]
#(257, 98, 2) => (128, 49, 64) => (64, 24, 128) => (32, 12, 128) => (16, 6, 256) => 12 labels
for i in range(5):
for j in range(num_convs[i]):
x = Conv2D(filters=num_filters[i], kernel_size=(3, 3), padding='same', activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = Flatten()(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.5)(x)
out = Dense(len(POSSIBLE_LABELS), activation='softmax')(x)
x = Dropout(dropout_rate, name='dp' + stage + '_1')(x)
x = Conv2D(num_filter, (kernel_size, kernel_size),
strides=strides,
activation=act,
name='conv' + stage + '_2',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(x)
x = Dropout(dropout_rate, name='dp' + stage + '_2')(x)
return x
num_filter = [12, 32, 48, 48, 32]
down_strides = (2, 2)
with tf.device('/cpu:0'):
inputs = Input(input_shape, name='main_input')
reduced = Dropout(0.2, name='dp_0')(inputs)
conv1_1 = standard_unit(reduced,
stage='11',
num_filter=num_filter[0],
strides=down_strides)
conv2_1 = standard_unit(conv1_1,
stage='21',
num_filter=num_filter[1],
strides=down_strides)
conv3_1 = standard_unit(conv2_1,
stage='31',
num_filter=num_filter[2],
strides=down_strides)
conv4_1 = standard_unit(conv3_1,
def Deeplab_xcep_pascal(weights=None,
input_tensor=None,
input_shape=(512, 512, 3),
num_classes=21,
backbone='xception',
OS=8,
model_path="",
activation=None):
if not (weights in {'pascalvoc', None}):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `pascalvoc` '
'(pre-trained on PascalVoc)')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
shape_before = tf.shape(x)
b4 = GlobalAveragePooling2D()(x)
# from (b_size, channels)->(b_size, 1, 1, channels)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation(tf.nn.relu)(b4)
# upsample. have to use compat because of the option align_corners
size_before = tf.keras.backend.int_shape(x)
b4 = Lambda(lambda x: tf.compat.v1.image.resize(
x, size_before[1:3], method='bilinear', align_corners=True))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation(tf.nn.relu, name='aspp0_activation')(b0)
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation(tf.nn.relu)(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
size_before2 = tf.keras.backend.int_shape(x)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, skip1.shape[1:3], method='bilinear', align_corners=True))(x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation(tf.nn.relu)(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
if (weights == 'pascalvoc' and num_classes == 21):
last_layer_name = 'logits_semantic'
else:
last_layer_name = 'custom_logits_semantic'
x = Conv2D(num_classes, (1, 1), padding='same', name=last_layer_name)(x)
size_before3 = tf.keras.backend.int_shape(img_input)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, size_before3[1:3], method='bilinear', align_corners=True))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
if activation in {'softmax', 'sigmoid'}:
x = tf.keras.layers.Activation(activation)(x)
model = Model(inputs, x, name='deeplabv3plus')
if weights == 'pascalvoc':
weights_path = model_path
model.load_weights(weights_path, by_name=True)
return model
def DIEN(
feature_dim_dict,
seq_feature_list,
embedding_size=8,
hist_len_max=16,
gru_type="GRU",
use_negsampling=False,
alpha=1.0,
use_bn=False,
hidden_size=(200, 80),
activation='sigmoid',
att_hidden_size=(64, 16),
att_activation=Dice,
att_weight_normalization=True,
l2_reg_deep=0,
l2_reg_embedding=1e-5,
final_activation='sigmoid',
keep_prob=1,
init_std=0.0001,
seed=1024,
):
"""Instantiates the Deep Interest Evolution Network architecture.
:param feature_dim_dict: dict,to indicate sparse field (**now only support sparse feature**)like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':[]}
:param seq_feature_list: list,to indicate sequence sparse field (**now only support sparse feature**),must be a subset of ``feature_dim_dict["sparse"]``
:param embedding_size: positive integer,sparse feature embedding_size.
:param hist_len_max: positive int, to indicate the max length of seq input
:param gru_type: str,can be GRU AIGRU AUGRU AGRU
:param use_negsampling: bool, whether or not use negtive sampling
:param alpha: float ,weight of auxiliary_loss
:param use_bn: bool. Whether use BatchNormalization before activation or not in deep net
:param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param activation: Activation function to use in deep net
:param att_hidden_size: list,list of positive integer , the layer number and units in each layer of attention net
:param att_activation: Activation function to use in attention net
:param att_weight_normalization: bool.Whether normalize the attention score of local activation unit.
:param l2_reg_deep: float. L2 regularizer strength applied to deep net
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:param keep_prob: float in (0,1]. keep_prob used in deep net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:return: A Keras model instance.
"""
check_feature_config_dict(feature_dim_dict)
sparse_input, dense_input, user_behavior_input, user_behavior_length = get_input(
feature_dim_dict, seq_feature_list, hist_len_max)
sparse_embedding_dict = {
feat.name:
Embedding(feat.dimension,
embedding_size,
embeddings_initializer=RandomNormal(mean=0.0,
stddev=init_std,
seed=seed),
embeddings_regularizer=l2(l2_reg_embedding),
name='sparse_emb_' + str(i) + '-' + feat.name)
for i, feat in enumerate(feature_dim_dict["sparse"])
}
query_emb_list = [
sparse_embedding_dict[feat](sparse_input[feat])
for feat in seq_feature_list
]
keys_emb_list = [
sparse_embedding_dict[feat](user_behavior_input[feat])
for feat in seq_feature_list
]
deep_input_emb_list = [
sparse_embedding_dict[feat.name](sparse_input[feat.name])
for feat in feature_dim_dict["sparse"]
]
query_emb = Concatenate()(
query_emb_list) if len(query_emb_list) > 1 else query_emb_list[0]
keys_emb = Concatenate()(
keys_emb_list) if len(keys_emb_list) > 1 else keys_emb_list[0]
deep_input_emb = Concatenate()(deep_input_emb_list) if len(
deep_input_emb_list) > 1 else deep_input_emb_list[0]
if use_negsampling:
neg_user_behavior_input = {
feat: Input(shape=(hist_len_max, ),
name='neg_seq_' + str(i) + '-' + feat)
for i, feat in enumerate(seq_feature_list)
}
neg_uiseq_embed_list = [
sparse_embedding_dict[feat](neg_user_behavior_input[feat])
for feat in seq_feature_list
]
neg_concat_behavior = Concatenate()(neg_uiseq_embed_list) if len(neg_uiseq_embed_list) > 1 else \
neg_uiseq_embed_list[0]
else:
neg_concat_behavior = None
hist, aux_loss_1 = interest_evolution(
keys_emb,
query_emb,
user_behavior_length,
gru_type=gru_type,
use_neg=use_negsampling,
neg_concat_behavior=neg_concat_behavior,
embedding_size=embedding_size,
att_hidden_size=att_hidden_size,
att_activation=att_activation,
att_weight_normalization=att_weight_normalization,
)
deep_input_emb = Concatenate()([deep_input_emb, hist])
deep_input_emb = tf.keras.layers.Flatten()(deep_input_emb)
if len(dense_input) > 0:
deep_input_emb = Concatenate()([deep_input_emb] +
list(dense_input.values()))
output = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input_emb)
final_logit = Dense(1, use_bias=False)(output)
output = PredictionLayer(final_activation)(final_logit)
model_input_list = get_inputs_list(
[sparse_input, dense_input, user_behavior_input])
if use_negsampling:
model_input_list += list(neg_user_behavior_input.values())
model_input_list += [user_behavior_length]
model = Model(inputs=model_input_list, outputs=output)
if use_negsampling:
model.add_loss(alpha * aux_loss_1)
return model
class_weight = compute_class_weight(class_weight='balanced',
classes=np.unique(train_data.label),
y=train_data.label)
# In[ ]:
# In[ ]:
# ## CNN模型
# In[ ]:
#CNN model
inputs = Input(shape=(48, 48, 1))
# First convolutional layer with ReLU-activation and max-pooling.
net = Conv2D(kernel_size=5,
strides=1,
filters=64,
padding='same',
activation='relu',
name='layer_conv1')(inputs)
net = MaxPooling2D(pool_size=2, strides=2)(net)
net = BatchNormalization(axis=-1)(net)
net = Dropout(0.25)(net)
# Second convolutional layer with ReLU-activation and max-pooling.
net = Conv2D(kernel_size=5,
strides=1,
def create_tf_genderestimation_facemobilenet(size, alpha):
input_tensor = Input(shape=(size, size, 3))
output_tensor = FaceMobileNet(input_tensor, alpha=alpha)
output_tensor = Dense(2)(output_tensor)
return Model(inputs=input_tensor, outputs=output_tensor)
def MTTS_CAN(n_frame,
nb_filters1,
nb_filters2,
input_shape,
kernel_size=(3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size=(2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape)
rawf_input = Input(shape=input_shape)
d1 = TSM_Cov2D(diff_input,
n_frame,
nb_filters1,
kernel_size,
padding='same',
activation='tanh')
d2 = TSM_Cov2D(d1,
n_frame,
nb_filters1,
kernel_size,
padding='valid',
activation='tanh')
r1 = Conv2D(nb_filters1, kernel_size, padding='same',
activation='tanh')(rawf_input)
r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1)
g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
gated1 = multiply([d2, g1])
d3 = AveragePooling2D(pool_size)(gated1)
d4 = Dropout(dropout_rate1)(d3)
r3 = AveragePooling2D(pool_size)(r2)
r4 = Dropout(dropout_rate1)(r3)
d5 = TSM_Cov2D(d4,
n_frame,
nb_filters2,
kernel_size,
padding='same',
activation='tanh')
d6 = TSM_Cov2D(d5,
n_frame,
nb_filters2,
kernel_size,
padding='valid',
activation='tanh')
r5 = Conv2D(nb_filters2, kernel_size, padding='same',
activation='tanh')(r4)
r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5)
g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
gated2 = multiply([d6, g2])
d7 = AveragePooling2D(pool_size)(gated2)
d8 = Dropout(dropout_rate1)(d7)
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(1, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(1, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model
def frcnn(config, stage='train'):
batch_size = config.IMAGES_PER_GPU
# 输入
input_image = Input(shape=config.IMAGE_INPUT_SHAPE, name='input_image')
input_image_meta = Input(shape=(12, ), name='input_image_meta')
gt_class_ids = Input(shape=(config.MAX_GT_INSTANCES, 1 + 1),
name='input_gt_class_ids')
gt_boxes = Input(shape=(config.MAX_GT_INSTANCES, 4 + 1),
name='input_gt_boxes')
# 特征及预测结果
features = config.base_fn(input_image)
boxes_regress, class_logits = rpn(features, config.RPN_ANCHOR_NUM)
# 生成anchor
anchors, anchors_tag = Anchor(config.RPN_ANCHOR_HEIGHTS,
config.RPN_ANCHOR_WIDTHS,
config.RPN_ANCHOR_BASE_SIZE,
config.RPN_ANCHOR_RATIOS,
config.RPN_ANCHOR_SCALES,
config.BACKBONE_STRIDE,
name='gen_anchors')(features)
# 裁剪到输入形状内
# anchors = UniqueClipBoxes(config.IMAGE_INPUT_SHAPE, name='clip_anchors')(anchors)
windows = Lambda(lambda x: x[:, 7:11])(input_image_meta)
# anchors = ClipBoxes()([anchors, windows])
# 应用分类和回归生成proposal
output_box_num = config.POST_NMS_ROIS_TRAINING if stage == 'train' else config.POST_NMS_ROIS_INFERENCE
iou_threshold = config.RPN_NMS_THRESHOLD_TRAINING if stage == 'train' else config.RPN_NMS_THRESHOLD_INFERENCE
proposal_boxes, _, _ = RpnToProposal(batch_size,
output_box_num=output_box_num,
iou_threshold=iou_threshold,
name='rpn2proposals')([
boxes_regress, class_logits,
anchors, anchors_tag,
input_image_meta
])
# proposal裁剪到图像窗口内
# proposal_boxes_coordinate, proposal_boxes_tag = Lambda(lambda x: [x[..., :4], x[..., 4:]])(proposal_boxes)
# proposal_boxes_coordinate = ClipBoxes()([proposal_boxes_coordinate, windows])
# proposal_boxes_coordinate = UniqueClipBoxes(config.IMAGE_INPUT_SHAPE,
# name='clip_proposals')(proposal_boxes_coordinate)
# 最后再合并tag返回
# proposal_boxes = Lambda(lambda x: tf.concat(x, axis=-1))([proposal_boxes_coordinate, proposal_boxes_tag])
if stage == 'train':
# 生成分类和回归目标
rpn_targets = RpnTarget(batch_size,
config.RPN_TRAIN_ANCHORS_PER_IMAGE,
name='rpn_target')([
gt_boxes, gt_class_ids, anchors,
anchors_tag
]) # [deltas,cls_ids,indices,..]
rpn_deltas, rpn_cls_ids, anchor_indices = rpn_targets[:3]
# 定义rpn损失layer
cls_loss_rpn = Lambda(lambda x: rpn_cls_loss(*x),
name='rpn_class_loss')(
[class_logits, rpn_cls_ids, anchor_indices])
regress_loss_rpn = Lambda(lambda x: rpn_regress_loss(*x),
name='rpn_bbox_loss')([
boxes_regress, rpn_deltas, anchor_indices
])
# 检测网络的分类和回归目标
detect_targets = DetectTarget(
batch_size,
config.TRAIN_ROIS_PER_IMAGE,
config.ROI_POSITIVE_RATIO,
name='rcnn_target')([gt_boxes, gt_class_ids, proposal_boxes])
roi_deltas, roi_class_ids, train_rois = detect_targets[:3]
# 检测网络
rcnn_deltas, rcnn_class_logits = rcnn(
features,
train_rois,
config.NUM_CLASSES,
config.IMAGE_MAX_DIM,
config.head_fn,
pool_size=config.POOL_SIZE,
fc_layers_size=config.RCNN_FC_LAYERS_SIZE)
# 检测网络损失函数
regress_loss_rcnn = Lambda(lambda x: detect_regress_loss(*x),
name='rcnn_bbox_loss')([
rcnn_deltas, roi_deltas, roi_class_ids
])
cls_loss_rcnn = Lambda(lambda x: detect_cls_loss(*x),
name='rcnn_class_loss')(
[rcnn_class_logits, roi_class_ids])
# 自定义度量命名
gt_num, positive_num, negative_num, rpn_miss_gt_num, rpn_gt_min_max_iou = rpn_targets[
3:]
rcnn_miss_gt_num, rcnn_miss_gt_num_as, gt_min_max_iou, pos_roi_num, neg_roi_num, roi_num = detect_targets[
3:]
gt_num = Lambda(lambda x: tf.identity(x),
name='identity_gt_num')(gt_num)
positive_num = Lambda(lambda x: tf.identity(x),
name='identity_positive_num')(positive_num)
negative_num = Lambda(lambda x: tf.identity(x),
name='identity_negative_num')(negative_num)
rpn_miss_gt_num = Lambda(
lambda x: tf.identity(x),
name='identity_rpn_miss_gt_num')(rpn_miss_gt_num)
rpn_gt_min_max_iou = Lambda(
lambda x: tf.identity(x),
name='identity_rpn_gt_min_max_iou')(rpn_gt_min_max_iou)
rcnn_miss_gt_num = Lambda(
lambda x: tf.identity(x),
name='identity_rcnn_miss_gt_num')(rcnn_miss_gt_num)
rcnn_miss_gt_num_as = Lambda(
lambda x: tf.identity(x),
name='identity_rcnn_miss_gt_num_as')(rcnn_miss_gt_num_as)
gt_min_max_iou = Lambda(lambda x: tf.identity(x),
name='identity_gt_min_max_iou')(gt_min_max_iou)
pos_roi_num = Lambda(lambda x: tf.identity(x),
name='identity_pos_roi_num')(pos_roi_num)
neg_roi_num = Lambda(lambda x: tf.identity(x),
name='identity_neg_roi_num')(neg_roi_num)
roi_num = Lambda(lambda x: tf.identity(x),
name='identity_roi_num')(roi_num)
# 构建模型
model = Model(
inputs=[input_image, input_image_meta, gt_class_ids, gt_boxes],
outputs=[
cls_loss_rpn, regress_loss_rpn, regress_loss_rcnn,
cls_loss_rcnn
] + [
gt_num, positive_num, negative_num, rpn_miss_gt_num,
rpn_gt_min_max_iou, roi_num, pos_roi_num, neg_roi_num,
rcnn_miss_gt_num, rcnn_miss_gt_num_as, gt_min_max_iou
]) # 在并行model中所有自定义度量必须在output中
# 多gpu训练
# if config.GPU_COUNT > 1:
# model = ParallelModel(model, config.GPU_LIST)
return model
else: # 测试阶段
# 检测网络
rcnn_deltas, rcnn_class_logits = rcnn(
features,
proposal_boxes,
config.NUM_CLASSES,
config.IMAGE_MAX_DIM,
config.head_fn,
pool_size=config.POOL_SIZE,
fc_layers_size=config.RCNN_FC_LAYERS_SIZE)
# 处理类别相关
rcnn_deltas = layers.Lambda(lambda x: deal_delta(*x),
name='deal_delta')(
[rcnn_deltas, rcnn_class_logits])
# 应用分类和回归生成最终检测框
detect_boxes, class_scores, detect_class_ids, detect_class_logits = ProposalToDetectBox(
score_threshold=config.DETECTION_MIN_CONFIDENCE,
output_box_num=config.DETECTION_MAX_INSTANCES,
iou_threshold=config.DETECTION_NMS_THRESHOLD,
name='proposals2detectboxes')(
[rcnn_deltas, rcnn_class_logits, proposal_boxes])
# 裁剪到窗口内部
detect_boxes_coordinate, detect_boxes_tag = Lambda(
lambda x: [x[..., :4], x[..., 4:]])(detect_boxes)
detect_boxes_coordinate = ClipBoxes()(
[detect_boxes_coordinate, windows])
# 最后再合并tag返回
detect_boxes = Lambda(lambda x: tf.concat(x, axis=-1))(
[detect_boxes_coordinate, detect_boxes_tag])
image_meta = Lambda(lambda x: tf.identity(x))(input_image_meta) # 原样返回
return Model(inputs=[input_image, input_image_meta],
outputs=[
detect_boxes, class_scores, detect_class_ids,
detect_class_logits, image_meta
])
def MT_Hybrid_CAN(n_frame,
nb_filters1,
nb_filters2,
input_shape_1,
input_shape_2,
kernel_size_1=(3, 3, 3),
kernel_size_2=(3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size_1=(2, 2, 2),
pool_size_2=(2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape_1)
rawf_input = Input(shape=input_shape_2)
# Motion branch
d1 = Conv3D(nb_filters1, kernel_size_1, padding='same',
activation='tanh')(diff_input)
d2 = Conv3D(nb_filters1, kernel_size_1, activation='tanh')(d1)
# App branch
r1 = Conv2D(nb_filters1, kernel_size_2, padding='same',
activation='tanh')(rawf_input)
r2 = Conv2D(nb_filters1, kernel_size_2, activation='tanh')(r1)
# Mask from App (g1) * Motion Branch (d2)
g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
g1 = K.expand_dims(g1, axis=-1)
gated1 = multiply([d2, g1])
# Motion Branch
d3 = AveragePooling3D(pool_size_1)(gated1)
d4 = Dropout(dropout_rate1)(d3)
d5 = Conv3D(nb_filters2, kernel_size_1, padding='same',
activation='tanh')(d4)
d6 = Conv3D(nb_filters2, kernel_size_1, activation='tanh')(d5)
# App branch
r3 = AveragePooling2D(pool_size_2)(r2)
r4 = Dropout(dropout_rate1)(r3)
r5 = Conv2D(nb_filters2, kernel_size_2, padding='same',
activation='tanh')(r4)
r6 = Conv2D(nb_filters2, kernel_size_2, activation='tanh')(r5)
# Mask from App (g2) * Motion Branch (d6)
g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
g2 = K.repeat_elements(g2, d6.shape[3], axis=-1)
g2 = K.expand_dims(g2, axis=-1)
gated2 = multiply([d6, g2])
# Motion Branch
d7 = AveragePooling3D(pool_size_1)(gated2)
d8 = Dropout(dropout_rate1)(d7)
# Motion Branch
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(n_frame, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(n_frame, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'facades'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
#self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', smoothL1],
loss_weights=[1, 100],
optimizer=optimizer)
################# Perceptual loss and L1 loss ######################
self.vggmodel=VGG19(weights="vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5",include_top=False)
#print(vggmodel.get_layer('block4_pool'))
#print(self.combined.output[1])
#print(vggmodel.get_layer('block4_pool').output)
#lossOut = vggmodel(inputs=self.combined.output[1], output = vggmodel.get_layer('block4_pool').output)
lossOut = self.vggmodel(inputs=self.combined.output[1])
self.vggmodel.trainable = False
for l in self.vggmodel.layers:
l.trainable = False
self.vgg_combined = Model(inputs=self.combined.input, outputs=lossOut)
self.vgg_combined.compile(loss='mse',optimizer='adam')
valid.trainable = False
def InceptionResNetV2(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception-ResNet v2 architecture.
Optionally loads weights pre-trained on ImageNet.
Note that when using TensorFlow, for best performance you should
set `"image_data_format": "channels_last"` in your Keras config
at `~/.keras/keras.json`.
The model and the weights are compatible with TensorFlow, Theano and
CNTK backends. The data format convention used by the model is
the one specified in your Keras config file.
Note that the default input image size for this model is 299x299, instead
of 224x224 as in the VGG16 and ResNet models. Also, the input preprocessing
function is different (i.e., do not use `imagenet_utils.preprocess_input()`
with this model. Use `preprocess_input()` defined in this module instead).
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is `False` (otherwise the input shape
has to be `(299, 299, 3)` (with `'channels_last'` data format)
or `(3, 299, 299)` (with `'channels_first'` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the last convolutional layer.
- `'avg'` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `'max'` means that global max pooling will be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is `True`, and
if no `weights` argument is specified.
Returns:
A Keras `Model` instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Stem block: 35 x 35 x 192
x = conv2d_bn(img_input, 32, 3, strides=2, padding='valid')
x = conv2d_bn(x, 32, 3, padding='valid')
x = conv2d_bn(x, 64, 3)
x = MaxPooling2D(3, strides=2)(x)
x = conv2d_bn(x, 80, 1, padding='valid')
x = conv2d_bn(x, 192, 3, padding='valid')
x = MaxPooling2D(3, strides=2)(x)
# Mixed 5b (Inception-A block): 35 x 35 x 320
branch_0 = conv2d_bn(x, 96, 1)
branch_1 = conv2d_bn(x, 48, 1)
branch_1 = conv2d_bn(branch_1, 64, 5)
branch_2 = conv2d_bn(x, 64, 1)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_pool = AveragePooling2D(3, strides=1, padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1)
branches = [branch_0, branch_1, branch_2, branch_pool]
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
x = Concatenate(axis=channel_axis, name='mixed_5b')(branches)
# 10x block35 (Inception-ResNet-A block): 35 x 35 x 320
for block_idx in range(1, 11):
x = inception_resnet_block(
x, scale=0.17, block_type='block35', block_idx=block_idx)
# Mixed 6a (Reduction-A block): 17 x 17 x 1088
branch_0 = conv2d_bn(x, 384, 3, strides=2, padding='valid')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 256, 3)
branch_1 = conv2d_bn(branch_1, 384, 3, strides=2, padding='valid')
branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_6a')(branches)
# 20x block17 (Inception-ResNet-B block): 17 x 17 x 1088
for block_idx in range(1, 21):
x = inception_resnet_block(
x, scale=0.1, block_type='block17', block_idx=block_idx)
# Mixed 7a (Reduction-B block): 8 x 8 x 2080
branch_0 = conv2d_bn(x, 256, 1)
branch_0 = conv2d_bn(branch_0, 384, 3, strides=2, padding='valid')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 288, 3, strides=2, padding='valid')
branch_2 = conv2d_bn(x, 256, 1)
branch_2 = conv2d_bn(branch_2, 288, 3)
branch_2 = conv2d_bn(branch_2, 320, 3, strides=2, padding='valid')
branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_2, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_7a')(branches)
# 10x block8 (Inception-ResNet-C block): 8 x 8 x 2080
for block_idx in range(1, 10):
x = inception_resnet_block(
x, scale=0.2, block_type='block8', block_idx=block_idx)
x = inception_resnet_block(
x, scale=1., activation=None, block_type='block8', block_idx=10)
# Final convolution block: 8 x 8 x 1536
x = conv2d_bn(x, 1536, 1, name='conv_7b')
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model
model = Model(inputs, x, name='inception_resnet_v2')
# Load weights
if weights == 'imagenet':
if include_top:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='e693bd0210a403b3192acc6073ad2e96')
else:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='d19885ff4a710c122648d3b5c3b684e4')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def get_unet(img_rows, img_cols, first_layer_num_filters=32, num_classes=1):
inputs = Input((img_rows, img_cols, 3))
###### ENCODER BRANCH ######
# leaky_relu = LeakyReLU(alpha=0.2)
leaky_relu = LeakyReLU(alpha=0.2)
first_conv_block = convolution_block(input_layer=inputs,
num_filters=first_layer_num_filters,
kernel_size=(3, 3),
activation_func=leaky_relu)
conv_ds_block_1 = conv_downsample_block(
input_layer=first_conv_block,
num_filters=first_layer_num_filters * 2,
kernel_size=(3, 3),
activation_func=leaky_relu,
max_pool_shape=(2, 2))
conv_ds_block_2 = conv_downsample_block(
input_layer=conv_ds_block_1,
num_filters=first_layer_num_filters * 4,
kernel_size=(3, 3),
activation_func=leaky_relu,
max_pool_shape=(2, 2))
conv_ds_block_3 = conv_downsample_block(
input_layer=conv_ds_block_2,
num_filters=first_layer_num_filters * 8,
kernel_size=(3, 3),
activation_func=leaky_relu,
max_pool_shape=(2, 2))
##### BOTTOM OF U-SHAPE #####
bottom_conv_block = conv_downsample_block(
input_layer=conv_ds_block_3,
num_filters=first_layer_num_filters * 16,
kernel_size=(3, 3),
activation_func=leaky_relu,
max_pool_shape=(2, 2))
###### DECODER BRANCH ######
conv_us_block_1 = conv_upsample_block(
input_layer=bottom_conv_block,
skip_connection_layer=conv_ds_block_3,
num_filters=first_layer_num_filters * 8,
kernel_size=(3, 3),
activation_func=leaky_relu,
upsampling_shape=(2, 2))
conv_us_block_2 = conv_upsample_block(
input_layer=conv_us_block_1,
skip_connection_layer=conv_ds_block_2,
num_filters=first_layer_num_filters * 4,
kernel_size=(3, 3),
activation_func=leaky_relu,
upsampling_shape=(2, 2))
conv_us_block_3 = conv_upsample_block(
input_layer=conv_us_block_2,
skip_connection_layer=conv_ds_block_1,
num_filters=first_layer_num_filters * 2,
kernel_size=(3, 3),
activation_func=leaky_relu,
upsampling_shape=(2, 2))
last_conv_block = conv_upsample_block(
input_layer=conv_us_block_3,
skip_connection_layer=first_conv_block,
num_filters=first_layer_num_filters,
kernel_size=(3, 3),
activation_func=leaky_relu,
upsampling_shape=(2, 2))
output_layer = Conv2D(num_classes, (1, 1),
activation='sigmoid')(last_conv_block)
model = Model(inputs=[inputs], outputs=[output_layer])
sgd_opt = SGD(lr=0.01, momentum=0.0, decay=0.0001, nesterov=False)
model.compile(optimizer=Adam(lr=1e-4, decay=1e-5),
loss=dice_coef_loss,
metrics=['accuracy']) # decay=1e-5
print(model.summary())
return model
def residual_sr(scale=2):
inputs = Input(shape=(None, None,
1)) # Input shape should be larger than 128,128,1
inputs_norm = Lambda(lambda t: t / 127.5 - 1)(inputs)
down_stack = [
downsample(64, 4, apply_batchnorm=False), # (bs, 64, 64, 64)
downsample(128, 4), # (bs, 32, 32, 128)
downsample(256, 4), # (bs, 16, 16, 256)
downsample(512, 4), # (bs, 8, 8, 512)
downsample(512, 4), # (bs, 4, 4, 512)
downsample(512, 4), # (bs, 2, 2, 512)
]
up_stack = [
upsample(512, 4, apply_dropout=True), # (bs, 2, 2, 1024)
upsample(512, 4, apply_dropout=True), # (bs, 4, 4, 1024)
upsample(512, 4, apply_dropout=True), # (bs, 8, 8, 1024)
upsample(256, 4), # (bs, 32, 32, 512)
upsample(128, 4), # (bs, 64, 64, 256)
upsample(64, 4), # (bs, 128, 128, 128)
]
initializer = tf.random_normal_initializer(0., 0.02)
last = tf.keras.layers.Conv2DTranspose(
OUTPUT_CHANNELS,
4,
strides=2,
padding='same',
kernel_initializer=initializer,
activation='tanh') # (bs, 128, 128, 3)
x = inputs_norm
# Downsampling through the model
skips = []
for down in down_stack:
x = down(x)
skips.append(x)
skips = reversed(skips[:-1])
# Upsampling and establishing the skip connections
for up, skip in zip(up_stack, skips):
x = up(x)
x = tf.keras.layers.Concatenate()([x, skip])
x = last(x)
x = x + inputs_norm
if scale == 2:
x = Conv2DTranspose(OUTPUT_CHANNELS,
4,
strides=2,
padding='same',
kernel_initializer=initializer)(x)
elif scale == 4:
x = Conv2DTranspose(OUTPUT_CHANNELS,
4,
strides=2,
padding='same',
kernel_initializer=initializer)(x)
x = Conv2DTranspose(OUTPUT_CHANNELS,
4,
strides=2,
padding='same',
kernel_initializer=initializer)(x)
x = Lambda(lambda t: (t + 1) * 127.5)(x)
return tf.keras.Model(inputs=inputs, outputs=x)
y_val_seq = y_tokenizer.texts_to_sequences(y_val)
y_train = pad_sequences(y_tr_seq, maxlen=max_len_summary, padding='post')
y_val = pad_sequences(y_val_seq, maxlen=max_len_summary, padding='post')
y_vocab = len(y_tokenizer.word_index) + 1
e = embeddings()
x_emb_weights = e.get_embedding_matrix(x_vocab, x_tokenizer)
y_emb_weights = e.get_embedding_matrix(y_vocab, y_tokenizer)
K.clear_session()
latent_dim = 300
embedding_dim = 50
# Encoder
encoder_inputs = Input(shape=(max_len_text, ))
#embedding layer
enc_emb = Embedding(x_vocab,
embedding_dim,
weights=[x_emb_weights],
trainable=False)(encoder_inputs)
#encoder lstm 1
encoder_lstm1 = LSTM(latent_dim,
return_sequences=True,
return_state=True,
dropout=0.4,
recurrent_dropout=0.4)
encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)
x_test_norm = np.expand_dims(x_test, axis=3) / 255
# %% add noise to data
noise_rate = 0.05
x_train_noisy = x_train_norm + np.random.choice([0, 1],
p=[1 - noise_rate, noise_rate],
size=x_train_norm.shape)
x_test_noisy = x_test_norm + np.random.choice([0, 1],
p=[1 - noise_rate, noise_rate],
size=x_test_norm.shape)
x_train_noisy = np.clip(x_train_noisy, 0, 1)
x_test_noisy = np.clip(x_test_noisy, 0, 1)
# %% model architecture
inputs = Input(shape=(28, 28, 1))
h = Conv2D(16, (3, 3), activation='relu', padding='same')(inputs)
h = MaxPooling2D((2, 2))(h)
h = Conv2D(8, (3, 3), activation='relu', padding='same')(h)
h = MaxPooling2D((2, 2))(h)
h = Conv2D(8, (3, 3), activation='relu', padding='same')(h)
encoded = MaxPooling2D((2, 2), padding='same')(h)
h = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)
h = UpSampling2D((2, 2))(h)
h = Conv2D(8, (3, 3), activation='relu', padding='same')(h)
h = UpSampling2D((2, 2))(h)
h = Conv2D(16, (3, 3), activation='relu')(h)
h = UpSampling2D((2, 2))(h)
def DIEN(feature_dim_dict, seq_feature_list, embedding_size=8, hist_len_max=16,
gru_type="GRU", use_negsampling=False, alpha=1.0, use_bn=False, dnn_hidden_units=(200, 80),
dnn_activation='relu',
att_hidden_units=(64, 16), att_activation="dice", att_weight_normalization=True,
l2_reg_dnn=0, l2_reg_embedding=1e-5, dnn_dropout=0, init_std=0.0001, seed=1024, task='binary'):
"""Instantiates the Deep Interest Evolution Network architecture.
:param feature_dim_dict: dict,to indicate sparse field (**now only support sparse feature**)like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':[]}
:param seq_feature_list: list,to indicate sequence sparse field (**now only support sparse feature**),must be a subset of ``feature_dim_dict["sparse"]``
:param embedding_size: positive integer,sparse feature embedding_size.
:param hist_len_max: positive int, to indicate the max length of seq input
:param gru_type: str,can be GRU AIGRU AUGRU AGRU
:param use_negsampling: bool, whether or not use negtive sampling
:param alpha: float ,weight of auxiliary_loss
:param use_bn: bool. Whether use BatchNormalization before activation or not in deep net
:param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN
:param dnn_activation: Activation function to use in DNN
:param att_hidden_units: list,list of positive integer , the layer number and units in each layer of attention net
:param att_activation: Activation function to use in attention net
:param att_weight_normalization: bool.Whether normalize the attention score of local activation unit.
:param l2_reg_dnn: float. L2 regularizer strength applied to DNN
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss
:return: A Keras model instance.
"""
check_feature_config_dict(feature_dim_dict)
sparse_input, dense_input, user_behavior_input, user_behavior_length = get_input(
feature_dim_dict, seq_feature_list, hist_len_max)
sparse_embedding_dict = {feat.name: Embedding(feat.dimension, embedding_size,
embeddings_initializer=RandomNormal(
mean=0.0, stddev=init_std, seed=seed),
embeddings_regularizer=l2(
l2_reg_embedding),
name='sparse_emb_' + str(i) + '-' + feat.name) for i, feat in
enumerate(feature_dim_dict["sparse"])}
query_emb_list = get_embedding_vec_list(sparse_embedding_dict,sparse_input,feature_dim_dict["sparse"],return_feat_list=seq_feature_list)
keys_emb_list = get_embedding_vec_list(sparse_embedding_dict,user_behavior_input,feature_dim_dict['sparse'],return_feat_list=seq_feature_list)
deep_input_emb_list = get_embedding_vec_list(sparse_embedding_dict, sparse_input, feature_dim_dict['sparse'])
query_emb = concat_fun(query_emb_list)
keys_emb = concat_fun(keys_emb_list)
deep_input_emb = concat_fun(deep_input_emb_list)
if use_negsampling:
neg_user_behavior_input = OrderedDict()
for i, feat in enumerate(seq_feature_list):
neg_user_behavior_input[feat] = Input(shape=(hist_len_max,), name='neg_seq_' + str(i) + '-' + feat)
neg_uiseq_embed_list = get_embedding_vec_list(sparse_embedding_dict,neg_user_behavior_input,feature_dim_dict["sparse"],seq_feature_list,)
# [sparse_embedding_dict[feat](
# neg_user_behavior_input[feat]) for feat in seq_feature_list]
neg_concat_behavior = concat_fun(neg_uiseq_embed_list)
else:
neg_concat_behavior = None
hist, aux_loss_1 = interest_evolution(keys_emb, query_emb, user_behavior_length, gru_type=gru_type,
use_neg=use_negsampling, neg_concat_behavior=neg_concat_behavior,
embedding_size=embedding_size, att_hidden_size=att_hidden_units,
att_activation=att_activation,
att_weight_normalization=att_weight_normalization, )
deep_input_emb = Concatenate()([deep_input_emb, hist])
deep_input_emb = tf.keras.layers.Flatten()(deep_input_emb)
if len(dense_input) > 0:
deep_input_emb = Concatenate()(
[deep_input_emb] + list(dense_input.values()))
output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn,
dnn_dropout, use_bn, seed)(deep_input_emb)
final_logit = Dense(1, use_bias=False)(output)
output = PredictionLayer(task)(final_logit)
model_input_list = get_inputs_list(
[sparse_input, dense_input, user_behavior_input])
if use_negsampling:
model_input_list += list(neg_user_behavior_input.values())
model_input_list += [user_behavior_length]
model = tf.keras.models.Model(inputs=model_input_list, outputs=output)
if use_negsampling:
model.add_loss(alpha * aux_loss_1)
tf.keras.backend.get_session().run(tf.global_variables_initializer())
return model
def _build_synthesis_network(config):
num_channels = 3
resolution_log2 = int(np.log2(config['resolution']))
num_layers = resolution_log2 * 2 - 2
num_styles = num_layers
# inputs
dlatents = Input([num_styles, config['dlatent_size']],
name='synthesis/dlatents')
# Noise inputs.
noise_inputs = []
for layer_idx in range(num_layers):
noise_inputs.append(
RandomNoise(name=f'synthesis/noise_{layer_idx}',
layer_idx=layer_idx)(dlatents))
# Things to do at the end of each layer.
def layer_epilogue(x, layer_idx, name):
name = 'G_synthesis/{}x{}/{}/'.format(x.shape[2], x.shape[2], name)
x = ApplyNoise(name=name + 'Noise')([x, noise_inputs[layer_idx]])
x = ApplyBias(name=name + 'bias')(x)
x = LeakyReLU(alpha=0.2, name=name + 'LeakyReLU')(x)
x = InstanceNorm(name=name + 'InstanceNorm')(x)
style = DenseLayer(units=x.shape[1] * 2,
gain=1,
name=name + 'StyleMod')(StridedSlice(
layer_idx,
name=name + 'StridedSlice')(dlatents))
x = StyleModApply(name=name + 'StyleModApply')([x, style])
return x
# Building blocks for remaining layers.
def block(res, x): # res = 3..resolution_log2
name, name0, name1 = '%dx%d' % (2**res, 2**
res), 'Conv0_up', 'Conv1'
# Conv0_up
upscaled = Upscale2d_conv2d(x,
name='G_synthesis/{}/{}'.format(
name, name0),
filters=nf(res - 1),
kernel_size=3,
use_bias=False)
x = layer_epilogue(
Blur(name='G_synthesis/{}/{}/Blur'.format(name, name0))(
upscaled), res * 2 - 4, name0)
# Conv1
x = layer_epilogue(
Conv2d(name='G_synthesis/{}/{}'.format(name, name1),
filters=nf(res - 1),
kernel_size=3,
use_bias=False)(x), res * 2 - 3, name1)
return x
def torgb(res, x): # res = 2..resolution_log2
lod = resolution_log2 - res
return Conv2d(name='G_synthesis/ToRGB_lod%d' % lod,
filters=num_channels,
kernel_size=1,
gain=1,
use_bias=True)(x)
# Early layers.
x = layer_epilogue(Const(name='G_synthesis/4x4/Const')(dlatents),
0,
name='Const')
x = layer_epilogue(
Conv2d(name='G_synthesis/4x4/Conv',
filters=nf(1),
kernel_size=3,
use_bias=False)(x), 1, 'Conv')
# Fixed structure: simple and efficient, but does not support progressive growing.
for res in range(3, resolution_log2 + 1):
x = block(res, x)
x = torgb(resolution_log2, x)
return Model(inputs=dlatents, outputs=x, name='G_synthesis')
