def _shortcut(input_feature, residual, conv_name_base=None, bn_name_base=None):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input_feature)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] /
residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input_feature
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
print('reshaping via a convolution...')
if conv_name_base is not None:
conv_name_base = conv_name_base + '1'
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001),
name=conv_name_base)(input_feature)
if bn_name_base is not None:
bn_name_base = bn_name_base + '1'
shortcut = BatchNormalization(axis=CHANNEL_AXIS,
name=bn_name_base)(shortcut)
return add([shortcut, residual])
def _shortcut(input, residual, weight_decay=1e-4):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = input.get_shape().as_list()
residual_shape = residual.get_shape().as_list()
stride_width = int(
round(input_shape[ROW_AXIS] /
residual_shape[ROW_AXIS])) # Problem for variable input
stride_height = int(round(input_shape[COL_AXIS] /
residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(weight_decay))(input)
return add([shortcut, residual])
def _conv_block(inp, convs, skip=True):
x = inp
count = 0
for conv in convs:
if count == (len(convs) - 2) and skip:
skip_connection = x
count += 1
if conv['stride'] > 1:
x = ZeroPadding2D(
((1, 0),
(1, 0)))(x) # peculiar padding as darknet prefer left and top
x = Conv2D(
conv['filter'],
conv['kernel'],
strides=conv['stride'],
padding='valid' if conv['stride'] > 1 else
'same', # peculiar padding as darknet prefer left and top
name='conv_' + str(conv['layer_idx']),
use_bias=False if conv['bnorm'] else True)(x)
if conv['bnorm']:
x = BatchNormalization(epsilon=0.001,
name='bnorm_' + str(conv['layer_idx']))(x)
if conv['leaky']:
x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)
return add([skip_connection, x]) if skip else x
def res_block(input_x, filters, sizes, layer_id):
f1, f2 = filters
s1, s2 = sizes
#1
x = Conv2D(f1,
s1,
padding='same',
use_bias=False,
name='conv_' + str(layer_id))(input_x)
x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(layer_id))(x)
x = LeakyReLU(alpha=0.1, name='leaky_' + str(layer_id))(x)
#2
x = Conv2D(f2,
s2,
padding='same',
use_bias=False,
name='conv_' + str(layer_id + 1))(x)
x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(layer_id + 1))(x)
x = LeakyReLU(alpha=0.1, name='leaky_' + str(layer_id + 1))(x)
x = add([input_x, x])
return x
def BasicBlock(inputs, filters, strides=1, downsample=None):
residual = inputs
out = TimeDistributed(
Conv2D(filters=filters,
kernel_size=3,
strides=strides,
padding='same',
kernel_initializer='he_uniform'))(inputs)
out = BatchNormalization()(out)
out = Activation('relu')(out)
out = TimeDistributed(
Conv2D(filters=filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='he_uniform'))(out)
out = BatchNormalization()(out)
if downsample is not None:
residual = downsample(inputs)
out = add([out, residual])
out = Activation('relu')(out)
return out
def _residual_block(x, filters, cfg):
# TODO: check activation
blk1 = Conv2D(filters=filters, kernel_size=4, strides=2, **cfg)(x)
blk2 = Conv2D(filters=filters, kernel_size=3, strides=1, **cfg)(blk1)
blk3 = Conv2D(filters=filters, kernel_size=3, strides=1, **cfg)(blk2)
output = add([blk1, blk3])
return output
def DenseBlock(self, inputs, outdim):
inputshape = K.int_shape(inputs)
bn = normalization.BatchNormalization(epsilon=2e-05, axis=3, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(inputs)
act = Activation('relu')(bn)
# act = Dropout(rate=0.2)(act)
conv1 = Conv2D(outdim, (3, 3), activation=None, padding='same', kernel_regularizer=regularizers.l2(0.0001))(act)
if inputshape[3] != outdim:
shortcut = Conv2D(outdim, (1, 1), padding='same', kernel_regularizer=regularizers.l2(0.0001))(inputs)
else:
shortcut = inputs
result1 = add([conv1, shortcut])
bn = normalization.BatchNormalization(epsilon=2e-05, axis=3, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(result1)
act = Activation('relu')(bn)
# act = Dropout(rate=0.2)(act)
conv2 = Conv2D(outdim, (3, 3), activation=None, padding='same', kernel_regularizer=regularizers.l2(0.0001))(act)
result = add([result1, conv2, shortcut])
result = Activation('relu')(result)
return result
def _res_func(x):
identity = Cropping2D(cropping=((2, 2), (2, 2)))(x)
a = Conv2D(nb_filter, (nb_row, nb_col),
strides=stride,
padding='valid')(x)
a = BatchNormalization()(a)
#a = LeakyReLU(0.2)(a)
a = Activation("relu")(a)
a = Conv2D(nb_filter, (nb_row, nb_col),
strides=stride,
padding='valid')(a)
y = BatchNormalization()(a)
return add([identity, y])
def ___conv4_block(input, k=1, dropout=0.0, se_net=False):
init = input
channel_axis = 1 if 'tf' == 'th' else -1
# Check if input number of filters is same as 64 * k, else
# create convolution2d for this input
if 'tf' == 'th':
if input.get_shape().as_list()[-1] != 64 * k:
init = Conv2D(64 * k, (1, 1),
activation='linear',
padding='same',
kernel_initializer=initia)(init)
else:
if input.get_shape().as_list()[-1] != 64 * k:
init = Conv2D(64 * k, (1, 1),
activation='linear',
padding='same',
kernel_initializer=initia)(init)
x = Conv2D(64 * k, (3, 3), padding='same',
kernel_initializer=initia)(input)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
if dropout > 0.0:
x = Dropout(dropout)(x)
x = Conv2D(64 * k, (3, 3), padding='same', kernel_initializer=initia)(x)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
if se_net:
x = squeeze_and_excitation_layer(x)
m = add([init, x])
return m
def get_Model(training):
input_shape = (img_w, img_h, 1) # (128, 64, 1)
# Make Networkw
inputs = Input(name='the_input', shape=input_shape,
dtype='float32') # (None, 128, 64, 1)
# Convolution layer (VGG)
inner = Conv2D(64, (3, 3),
padding='same',
name='conv1',
kernel_initializer='he_normal')(
inputs) # (None, 128, 64, 64)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max1')(inner) # (None,64, 32, 64)
inner = Conv2D(128, (3, 3),
padding='same',
name='conv2',
kernel_initializer='he_normal')(
inner) # (None, 64, 32, 128)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max2')(inner) # (None, 32, 16, 128)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv3',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv4',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max3')(inner) # (None, 32, 8, 256)
inner = Conv2D(512, (3, 3),
padding='same',
name='conv5',
kernel_initializer='he_normal')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(512, (3, 3), padding='same',
name='conv6')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max4')(inner) # (None, 32, 4, 512)
inner = Conv2D(512, (2, 2),
padding='same',
kernel_initializer='he_normal',
name='con7')(inner) # (None, 32, 4, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
# CNN to RNN
inner = Reshape(target_shape=((32, 2048)),
name='reshape')(inner) # (None, 32, 2048)
inner = Dense(64,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(inner) # (None, 32, 64)
# RNN layer
lstm_1 = LSTM(256,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm1')(inner) # (None, 32, 512)
lstm_1b = LSTM(256,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm1_b')(inner)
reversed_lstm_1b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_1b)
lstm1_merged = add([lstm_1, reversed_lstm_1b]) # (None, 32, 512)
lstm1_merged = BatchNormalization()(lstm1_merged)
lstm_2 = LSTM(256,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm2')(lstm1_merged)
lstm_2b = LSTM(256,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm2_b')(lstm1_merged)
reversed_lstm_2b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_2b)
lstm2_merged = concatenate([lstm_2, reversed_lstm_2b]) # (None, 32, 1024)
lstm2_merged = BatchNormalization()(lstm2_merged)
# transforms RNN output to character activations:
inner = Dense(num_classes, kernel_initializer='he_normal',
name='dense2')(lstm2_merged) #(None, 32, 63)
y_pred = Activation('softmax', name='softmax')(inner)
labels = Input(name='the_labels', shape=[max_text_len],
dtype='float32') # (None ,8)
input_length = Input(name='input_length', shape=[1],
dtype='int64') # (None, 1)
label_length = Input(name='label_length', shape=[1],
dtype='int64') # (None, 1)
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(focal_ctc_lambda_func, output_shape=(1, ),
name='ctc')([labels, y_pred, input_length,
label_length]) #(None, 1)
if training:
return Model(inputs=[inputs, labels, input_length, label_length],
outputs=loss_out)
else:
return Model(inputs=[inputs], outputs=y_pred)
for word, i in wordtoix.items():
#if i < max_words:
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# Words not found in the embedding index will be all zeros
embedding_matrix[i] = embedding_vector
#构建一个lstm神经网络
inputs1 = Input(shape=(2048, )) #Input():用来实例化一个keras张量
fe1 = Dropout(0.4)(inputs1)
fe2 = Dense(256, activation='relu')(fe1)
inputs2 = Input(shape=(max_length, ))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
se3 = LSTM(256)(se2)
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
outputs = Dense(vocab_size, activation='softmax')(decoder2)
model = Model(inputs=[inputs1, inputs2], outputs=outputs)
#这里还可以看模型的总结
#model.summary()
model.layers[2].set_weights([embedding_matrix
]) #embedding matrix就是embedding层的参数
model.layers[2].trainable = False
model.compile(loss='categorical_crossentropy',
optimizer='adam') #使用交叉熵作为目标函数，梯度下降算法
epochs = 10
number_pics_per_bath = 3
def _residual_block(x, filters, cfg, stride=2):
blk1 = Conv2D(filters=filters, kernel_size=4, strides=stride, **cfg)(x)
blk2 = Conv2D(filters=filters, kernel_size=3, strides=1, **cfg)(blk1)
blk3 = Conv2D(filters=filters, kernel_size=3, strides=1, **cfg)(blk2)
output = add([blk1, blk3])
return output
def _model_construction_test(self, is_training=True):
# Model
encoder_inputs = Input(shape=(self._seq_len, self._input_dim),
name='encoder_input')
encoder_outputs, enc_state_h, enc_state_c = Residual_enc(
encoder_inputs,
rnn_unit=self._rnn_units,
rnn_depth=self._rnn_layers,
rnn_dropout=self._drop_out)
encoder_states = [enc_state_h, enc_state_c]
decoder_inputs = Input(shape=(None, self._output_dim),
name='decoder_input')
layers_dec, decoder_outputs, dec_state_h, dec_state_c = Residual_dec(
decoder_inputs,
rnn_unit=self._rnn_units,
rnn_depth=self._rnn_layers,
rnn_dropout=self._drop_out,
init_states=encoder_states)
attn_layer = AttentionLayer(input_shape=([
self._batch_size, self._seq_len, self._rnn_units
], [self._batch_size, self._seq_len, self._rnn_units]),
name='attention_layer')
attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])
decoder_outputs = Concatenate(
axis=-1, name='concat_layer')([decoder_outputs, attn_out])
# dense decoder_outputs
decoder_dense = Dense(self._output_dim, activation='relu')
decoder_outputs = decoder_dense(decoder_outputs)
# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
if is_training:
return model
else:
self._logger.info("Load model from: {}".format(self._log_dir))
model.load_weights(self._log_dir + 'best_model.hdf5')
model.compile(optimizer=self._optimizer,
loss='mse',
metrics=['mse', 'mae'])
# --------------------------------------- ENcoder model ----------------------------------------------------
self.encoder_model = Model(encoder_inputs,
[encoder_outputs] + encoder_states)
# plot_model(model=self.encoder_model, to_file=self._log_dir + '/encoder.png', show_shapes=True)
# --------------------------------------- Decoder model ----------------------------------------------------
decoder_state_input_h = Input(shape=(self._rnn_units, ),
name='decoder_state_input_h')
decoder_state_input_c = Input(shape=(self._rnn_units, ),
name='decoder_state_input_c')
decoder_states_inputs = [
decoder_state_input_h, decoder_state_input_c
]
decoder_outputs, _, _ = layers_dec[0](
decoder_inputs, initial_state=decoder_states_inputs)
for i in range(1, self._rnn_layers):
d_o, dec_state_h, dec_state_c = layers_dec[i](decoder_outputs)
decoder_outputs = add([decoder_outputs, d_o])
decoder_states = [dec_state_h, dec_state_c]
encoder_inf_states = Input(shape=(self._seq_len, self._rnn_units),
name='encoder_inf_states_input')
attn_out, attn_states = attn_layer(
[encoder_inf_states, decoder_outputs])
decoder_outputs = Concatenate(
axis=-1, name='concat')([decoder_outputs, attn_out])
decoder_dense = Dense(self._output_dim, activation='relu')
decoder_outputs = decoder_dense(decoder_outputs)
self.decoder_model = Model([decoder_inputs, encoder_inf_states] +
decoder_states_inputs,
[decoder_outputs] + decoder_states)
# plot_model(model=self.decoder_model, to_file=self._log_dir + '/decoder.png', show_shapes=True)
return model
def layer(x):
x = add([left(x), right(x)])
x = Activation('relu')(x)
return x