def build_single_unet_per_channel(input_shape,
last_activation,
n_depth=2,
n_filter_base=16,
kernel_size=(3, 3, 3),
n_conv_per_depth=2,
activation="relu",
batch_norm=False,
dropout=0.0,
pool_size=(2, 2, 2),
residual=False,
prob_out=False,
eps_scale=1e-3):
""" TODO """
if last_activation is None:
raise ValueError(
"last activation has to be given (e.g. 'sigmoid', 'relu')!")
all((s % 2 == 1 for s in kernel_size)) or _raise(
ValueError('kernel size should be odd in all dimensions.'))
channel_axis = -1 if backend_channels_last() else 1
n_dim = len(kernel_size)
conv = Conv2D if n_dim == 2 else Conv3D
num_channels = input_shape[channel_axis]
input = Input(input_shape, name="input")
out_channels = []
num_channel_out = 1
for i in range(num_channels):
c = Lambda(lambda x: x[:, ..., i:i + 1])(input)
unet = unet_block(n_depth,
n_filter_base,
kernel_size,
activation=activation,
dropout=dropout,
batch_norm=batch_norm,
n_conv_per_depth=n_conv_per_depth,
pool=pool_size,
prefix='channel_{}'.format(i))(c)
final = conv(num_channel_out, (1, ) * n_dim, activation='linear')(unet)
if residual:
if not (num_channel_out == 1
if backend_channels_last() else num_channel_out == 1):
raise ValueError(
"number of input and output channels must be the same for a residual net."
)
final = Add()([final, input])
final = Activation(activation=last_activation)(final)
if prob_out:
scale = conv(num_channel_out, (1, ) * n_dim,
activation='softplus')(unet)
scale = Lambda(lambda x: x + np.float32(eps_scale))(scale)
final = Concatenate(axis=channel_axis)([final, scale])
out_channels.append(final)
if len(out_channels) > 1:
output = Concatenate(axis=channel_axis)(out_channels)
return Model(inputs=input, outputs=output)
else:
return Model(inputs=input, outputs=out_channels[0])
def Attention_Wavenet(n_steps, n_features, activation):
# convolutional operation parameters
n_filters = 32
filter_width = 2
dilation_rates = [2**i for i in range(9)] * 3
# define an input history series and pass it through a stack of dilated causal convolution blocks.
history_seq = Input(shape=[n_steps, n_features])
x = history_seq
skips = []
for dilation_rate in dilation_rates:
# preprocessing - equivalent to time-distributed dense
x = Conv1D(16, 1, padding='same', activation=activation)(x)
# filter convolution
x_f = Conv1D(filters=n_filters,
kernel_size=filter_width,
padding='causal',
dilation_rate=dilation_rate)(x)
x_f = Dropout(0.1)(x_f)
# gating convolution
x_g = Conv1D(filters=n_filters,
kernel_size=filter_width,
padding='causal',
dilation_rate=dilation_rate)(x)
x_g = Dropout(0.1)(x_g)
# multiply filter and gating branches
z = Multiply()([Activation('tanh')(x_f), Activation('sigmoid')(x_g)])
# postprocessing - equivalent to time-distributed dense
z = Conv1D(16, 1, padding='same', activation=activation)(z)
# residual connection
x = Add()([x, z])
# collect skip connections
skips.append(z)
# add all skip connection outputs
out = Activation(activation)(Add()(skips))
out = Attention(use_scale=True, causal=True)([out, out])
# final time-distributed dense layers
out = Conv1D(16, 1, activation=activation, padding='same')(out)
out = MaxPool1D(pool_size=2)(out)
out = keras.layers.Flatten()(out)
out = Dense(get_output_dim(n_steps * n_features),
kernel_initializer='lecun_normal',
activation='selu')(out)
encoder = Model(inputs=[history_seq], outputs=[out])
decoder = keras.models.Sequential([
keras.layers.Reshape(
[get_output_dim(n_steps * n_features), 1, 1],
input_shape=[get_output_dim(n_steps * n_features)]),
Conv2DTranspose(filters=32, kernel_size=3, activation=activation),
Conv2DTranspose(filters=16, kernel_size=3, activation=activation),
keras.layers.Flatten(),
Dense(n_steps * n_features),
keras.layers.Reshape([n_steps, n_features])
])
return encoder, decoder
def create_uNet(input_shape,
nclasses,
filters=[30, 45, 60],
lambda_regularization=None,
activation='elu'):
if lambda_regularization is not None:
lambda_regularization = keras.regularizers.l2(lambda_regularization)
tensor_list = []
input_tensor = Input(shape=input_shape, name="input")
# you could make a loop
# to append and pop
# 256x256
tensor = Convolution2D(filters[0],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(input_tensor)
tensor = Convolution2D(filters[1],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
#############################
tensor_list.append(tensor)
tensor = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding='same')(tensor)
# 128x128
tensor = Convolution2D(filters[2],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
tensor = Convolution2D(filters[1],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
tensor_list.append(tensor)
tensor = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding='same')(tensor)
# 64x64
tensor = Convolution2D(filters[2],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
tensor = Convolution2D(filters[1],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
# upsample
# 128x128
tensor = UpSampling2D(size=2)(
tensor) # take 1 pixel and expand it out to 2 x 2
tensor = Convolution2D(filters[1],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
tensor = Add()([tensor, tensor_list.pop()])
# upsample
tensor = UpSampling2D(size=2)(
tensor) # take 1 pixel and expand it out to 2 x 2
# 256 x 256
tensor = Convolution2D(filters[1],
kernel_size=(3, 3),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation=activation)(tensor)
tensor = Add()([tensor, tensor_list.pop()])
#############################
output_tensor = Convolution2D(nclasses,
kernel_size=(1, 1),
padding='same',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='zeros',
kernel_regularizer=lambda_regularization,
activation='softmax',
name='output')(tensor)
model = Model(inputs=input_tensor, outputs=output_tensor)
opt = keras.optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
optimizer=opt,
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
return model
# The discriminator_model is more complex. It takes both real image samples and random
# noise seeds as input. The noise seed is run through the generator model to get
# generated images. Both real and generated images are then run through the
# discriminator. Although we could concatenate the real and generated images into a
# single tensor, we don't (see model compilation for why).
real_samples = Input(shape=X_train.shape[1:])
generator_input_for_discriminator = Input(shape=(100, ))
generated_samples_for_discriminator = generator(
generator_input_for_discriminator)
discriminator_output_from_generator = discriminator(
generated_samples_for_discriminator)
discriminator_output_from_real_samples = discriminator(real_samples)
# We also need to generate weighted-averages of real and generated samples,
# to use for the gradient norm penalty.
averaged_samples = Add()([real_samples, generated_samples_for_discriminator])
# We then run these samples through the discriminator as well. Note that we never
# really use the discriminator output for these samples - we're only running them to
# get the gradient norm for the gradient penalty loss.
averaged_samples_out = discriminator(averaged_samples)
# The gradient penalty loss function requires the input averaged samples to get
# gradients. However, Keras loss functions can only have two arguments, y_true and
# y_pred. We get around this by making a partial() of the function with the averaged
# samples here.
partial_gp_loss = partial(gradient_penalty_loss,
averaged_samples=averaged_samples,
gradient_penalty_weight=GRADIENT_PENALTY_WEIGHT)
# Functions need names or Keras will throw an error
partial_gp_loss.__name__ = 'gradient_penalty'
# convolution을 연결할 때 (some possible problems: gradient vanishing)
# input(784) -> Dense(32)- relu--> * Dense(32) -relu- -> Dense(32) -relu- -> Dense(10) -softmax-
# 필터를 통과할 수록 이미지의 크기가 작아진다
# ResidualNet은 * 와 같은 지점에서 건너 띄어서 바로 출력층으로 가는 것?
# input(784) -> Dense(32)- relu - + add -> Dense(10) -softmax-
# 지름길로 가버리는 것
# + add 가 일어난다
# + 와 concatenate을 구별해야한다
# + : element, element끼리 더하기 -> 32가 입력으로 들어온다 (더하기의 개념) -> 32개와 32개를 더해줌 i.e. ((1,2),(3,4)) + ((5,6),(7,8)) = ((6, 8), (10, 12))
# concatenate: 더해주는 것 -> 64개가 입력으로 들어오는 것 i.e. ((1,2),(3,4)) + ((5,6),(7,8)) = ((1,2),(3,4),(5,6),(7,8))
input_tensor = Input(shape=(784, ))
sc = Dense(32,
activation='relu')(input_tensor) # 첫번째 Dense를 지나서 나온 출력값을 sc라고 부름
x = Dense(32, activation='relu')(sc)
x = Dense(32, activation='relu')(x)
x = Add()([x,
sc]) # concatenate은 함수라서 () 에 값을 줬지만, Add는 클래스이름이라 먼저 호출 하고 뒤에 붙여줌
output_tensor = Dense(10, 'softmax')(x)
# 개념적으로 구현해 보았다
# Model 생성
model = Model(input_tensor, output_tensor)
model.summary()
# n_input * n_output + n_b
# 784 * 32 + 32 = 25120
# 32 * 32 + 32 = 1056
# 신경망이 깊어지면 깊어질 수록 기울기가 소실된다는 문제가 있었다
# 지름길을 통과하는 네트워크를 하나를 만들어서 기울기 손실이 없는 네트워크 한개를 끝까지 보내면 -> 마지막 단계에서 소실되어도 모든 망을 통과한 값 + 소실되지 않은 값을 더해줘서 계산해주는
def getResidualBlock(I, filter_size, featmaps, stage, block, shortcut,
convArgs, bnArgs, d):
"""Get residual block."""
activation = d.act
drop_prob = d.dropout
nb_fmaps1, nb_fmaps2 = featmaps
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
#if K.image_data_format() == 'channels_first' and K.ndim(I) != 3:
# channel_axis = 1
#else:
channel_axis = -1
if d.model == "real":
O = BatchNormalization(name=bn_name_base + '_2a', **bnArgs)(I)
elif d.model == "complex":
O = ComplexBN(name=bn_name_base + '_2a', **bnArgs)(I)
O = Spline()(O) #Activation(activation)(O)
if shortcut == 'regular' or d.spectral_pool_scheme == "nodownsample":
if d.model == "real":
O = Conv2D(nb_fmaps1,
filter_size,
name=conv_name_base + '2a',
**convArgs)(O)
elif d.model == "complex":
O = ComplexConv2D(nb_fmaps1,
filter_size,
name=conv_name_base + '2a',
**convArgs)(O)
elif shortcut == 'projection':
if d.spectral_pool_scheme == "proj":
O = applySpectralPooling(O, d)
if d.model == "real":
O = Conv2D(nb_fmaps1,
filter_size,
name=conv_name_base + '2a',
strides=(2, 2),
**convArgs)(O)
elif d.model == "complex":
O = ComplexConv2D(nb_fmaps1,
filter_size,
name=conv_name_base + '2a',
strides=(2, 2),
**convArgs)(O)
if d.model == "real":
O = BatchNormalization(name=bn_name_base + '_2b', **bnArgs)(O)
O = Spline()(O) #Activation(activation)(O)
O = Conv2D(nb_fmaps2,
filter_size,
name=conv_name_base + '2b',
**convArgs)(O)
elif d.model == "complex":
O = ComplexBN(name=bn_name_base + '_2b', **bnArgs)(O)
O = Spline()(O) #Activation(activation)(O)
O = ComplexConv2D(nb_fmaps2,
filter_size,
name=conv_name_base + '2b',
**convArgs)(O)
if shortcut == 'regular':
O = Add()([O, I])
elif shortcut == 'projection':
if d.spectral_pool_scheme == "proj":
I = applySpectralPooling(I, d)
if d.model == "real":
X = Conv2D(
nb_fmaps2, (1, 1),
name=conv_name_base + '1',
strides=(2, 2) if d.spectral_pool_scheme != "nodownsample" else
(1, 1),
**convArgs)(I)
O = Concatenate(channel_axis)([X, O])
elif d.model == "complex":
X = ComplexConv2D(
nb_fmaps2, (1, 1),
name=conv_name_base + '1',
strides=(2, 2) if d.spectral_pool_scheme != "nodownsample" else
(1, 1),
**convArgs)(I)
O_real = Concatenate(channel_axis)(
[X[..., :X.shape[-1] // 2], O[..., :O.shape[-1] // 2]])
O_imag = Concatenate(channel_axis)(
[X[..., X.shape[-1] // 2:], O[..., O.shape[-1] // 2:]])
O = Concatenate(channel_axis)([O_real, O_imag])
return O
def _make_block_basic(
self, input_tensor, first_block=True, filters=64, stride=2, radix=1, avd=False, avd_first=False, is_first=False
):
"""Conv2d_BN_Relu->Bn_Relu_Conv2d
"""
x = input_tensor
x = BatchNormalization(axis=self.channel_axis, epsilon=1.001e-5)(x)
x = Activation(self.active)(x)
short_cut = x
inplanes = input_tensor.shape[-1]
if stride != 1 or inplanes != filters * self.block_expansion:
if self.avg_down:
if self.dilation == 1:
short_cut = AveragePooling2D(pool_size=stride, strides=stride, padding="same", data_format="channels_last")(
short_cut
)
else:
short_cut = AveragePooling2D(pool_size=1, strides=1, padding="same", data_format="channels_last")(short_cut)
short_cut = Conv2D(
filters,
kernel_size=1,
strides=1,
padding="same",
kernel_initializer="he_normal",
use_bias=False,
data_format="channels_last",
)(short_cut)
else:
short_cut = Conv2D(
filters,
kernel_size=1,
strides=stride,
padding="same",
kernel_initializer="he_normal",
use_bias=False,
data_format="channels_last",
)(short_cut)
group_width = int(filters * (self.bottleneck_width / 64.0)) * self.cardinality
avd = avd and (stride > 1 or is_first)
avd_first = avd_first
if avd:
avd_layer = AveragePooling2D(pool_size=3, strides=stride, padding="same", data_format="channels_last")
stride = 1
if avd and avd_first:
x = avd_layer(x)
if radix >= 1:
x = self._SplAtConv2d(
x,
filters=group_width,
kernel_size=3,
stride=stride,
dilation=self.dilation,
groups=self.cardinality,
radix=radix,
)
else:
x = Conv2D(
filters,
kernel_size=3,
strides=stride,
padding="same",
kernel_initializer="he_normal",
dilation_rate=self.dilation,
use_bias=False,
data_format="channels_last",
)(x)
if avd and not avd_first:
x = avd_layer(x)
# print('can')
x = BatchNormalization(axis=self.channel_axis, epsilon=1.001e-5)(x)
x = Activation(self.active)(x)
x = Conv2D(
filters,
kernel_size=3,
strides=1,
padding="same",
kernel_initializer="he_normal",
dilation_rate=self.dilation,
use_bias=False,
data_format="channels_last",
)(x)
m2 = Add()([x, short_cut])
return m2
filters=12,
kernel_size=(1, 1),
padding='valid',
data_format='channels_last',
use_bias=True)(B2)
B3 = LeakyReLU()(B3)
B4 = LocallyConnected2D(name="layerB4",
filters=15,
kernel_size=(2, 3),
strides=(2, 2),
padding='valid',
data_format='channels_last',
activation=None,
use_bias=True)(B3)
C = Add()([B4, in1merge])
C = LeakyReLU()(C)
#D = LocallyConnected2D( name="layerD", filters=10, kernel_size=(1,1), strides=(1,1), padding='valid', data_format='channels_last', use_bias=True )( C )
#D = LeakyReLU()( D )
E = LocallyConnected2D(name="layerE",
filters=15,
kernel_size=(2, 1),
strides=(2, 1),
padding='valid',
data_format='channels_last',
use_bias=True)(C)
E = LeakyReLU()(E)
#Epool = MaxPooling2D(pool_size=(2, 1), strides=(2,1), padding='valid', data_format='channels_last')( E )
#print( Epool.shape )
F = LocallyConnected2D(name="layerF",
def convolutional_block(X, f, filters, stage, block, s=2):
"""
Implementation of the convolutional block as defined in Figure 4
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
f -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
s -- Integer, specifying the stride to be used
Returns:
X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# Retrieve Filters
F1, F2, F3 = filters
# Save the input value
X_shortcut = X
##### MAIN PATH #####
# First component of main path
X = Conv2D(F1, (1, 1),
strides=(s, s),
name=conv_name_base + '2a',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
X = Activation('relu')(X)
# Second component of main path (≈3 lines)
X = Conv2D(filters=F2,
kernel_size=(f, f),
strides=(1, 1),
padding='same',
name=conv_name_base + '2b',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
X = Activation('relu')(X)
# Third component of main path (≈2 lines)
X = Conv2D(filters=F3,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
name=conv_name_base + '2c',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)
##### SHORTCUT PATH #### (≈2 lines)
X_shortcut = Conv2D(filters=F3,
kernel_size=(1, 1),
strides=(s, s),
padding='valid',
name=conv_name_base + '1',
kernel_initializer=glorot_uniform(seed=0))(X_shortcut)
X_shortcut = BatchNormalization(axis=3,
name=bn_name_base + '1')(X_shortcut)
# Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
return X
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Activation, Concatenate, Add, Multiply
from unit_test.helper import tf_random_seed
model_list = []
for activation_item in ['relu', 'sigmoid', 'softmax']:
for merge_item in [Concatenate(axis=-1), Add(), Multiply()]:
tf_random_seed()
placeholder = Input((32, 32, 3), name='data')
x_0 = Activation(activation_item)(placeholder)
x_1 = Activation(activation_item)(placeholder)
x = merge_item([x_0, x_1])
model = Model(
placeholder,
x,
name=
f'model_single_layer_{activation_item}_{merge_item.__class__.__name__.lower()}'
)
model_list.append(model)
def fireModule(x,
squeeze=16,
expand=64,
kernels=((1, 1), (3, 3)),
bypass=False,
complexBypass=False):
if (len(kernels) != 2):
raise Exception(
'The length of kernels must equal 2\nGiven kernel: %s' %
str(kernels))
lowKernel = kernels[0]
highKernel = kernels[1]
y = Conv2D(squeeze,
lowKernel,
padding='same',
name='%d_%dConv%dSqueeze' %
(lowKernel[0], lowKernel[1], callCounter.counter))(x)
y = Activation('relu', name='Relu1%d' % callCounter.counter)(y)
left = Conv2D(expand,
highKernel,
padding='same',
name='%d_%dConv%dLeft' %
(highKernel[0], highKernel[1], callCounter.counter))(y)
left = Activation('relu', name='Relu2%d' % callCounter.counter)(left)
right = Conv2D(expand,
lowKernel,
padding='same',
name='%d_%dConv%dRight' %
(lowKernel[0], lowKernel[1], callCounter.counter))(y)
right = Activation('relu', name='Relu3%d' % callCounter.counter)(right)
concat = Concatenate(axis=-1)([left, right])
if (complexBypass):
additionalConv = Conv2D(expand * 2, (1, 1),
name='1_1AdditionalConv%d' %
callCounter.counter)(x)
concat = Add()([concat, additionalConv])
elif (bypass):
if (str(x.shape) != str(concat.shape)):
raise Exception(
'The shape of input and output must be same.\nInput shape: %s\nOutput shape: %s'
% (str(x.shape), str(concat.shape)))
concat = Add(name='Add%d' % callCounter.counter)([concat, x])
callCounter.counter += 1
return concat
def Segnet(height=576,width=576,channel=3,n_labels=2):
input_shape = (height, width, channel)
kernel = 3
args = {"ksize": (1,2,2,1), "strides":(1,2,2,1)}
pool_size = (1,2,2,1)
inputs = Input(input_shape)
x = Conv2D(64, (kernel, kernel), padding="same",name="conv_1_1")(inputs)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(64, (kernel, kernel), padding="same",name="conv_1_2")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
pool_1, mask_1 = Lambda(MaxPool2DWithArgmax, arguments=args)(x)
x = Conv2D(128, (kernel, kernel), padding="same",name="conv_2_1")(pool_1)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(128, (kernel, kernel), padding="same",name="conv_2_2")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
model_A = Model(inputs=inputs,outputs=x)
model_B = input_other()
# combined = Concatenate()([model_A.output,model_B.output])
combined = Add()([model_A.output,model_B.output])
x = combined
pool_2, mask_2 = Lambda(MaxPool2DWithArgmax, arguments=args)(x)
x = Conv2D(256, (kernel, kernel), padding="same")(pool_2)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(256, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(256, (1, 1), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
pool_3, mask_3 = Lambda(MaxPool2DWithArgmax, arguments=args)(x)
x = Conv2D(512, (kernel, kernel), padding="same")(pool_3)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (1, 1), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
pool_4, mask_4 = Lambda(MaxPool2DWithArgmax, arguments=args)(x)
x = Conv2D(512, (kernel, kernel), padding="same")(pool_4)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (1, 1), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
pool_5, mask_5 = Lambda(MaxPool2DWithArgmax, arguments=args)(x)
# ======================================================================
x = Lambda(Unpool2D, arguments={"factor": pool_size})([pool_5, mask_5])
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_4])
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(512, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(256, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_3])
x = Conv2D(256, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(256, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(128, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_2])
x = Conv2D(128, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(64, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_1])
x = Conv2D(64, (kernel, kernel), padding="same")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(n_labels, (1, 1), padding="valid")(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
# x = Reshape(
# (input_shape[0] * input_shape[1], n_labels),
# input_shape=(input_shape[0], input_shape[1], n_labels),
# )(x)
outputs = Activation("softmax")(x)
return Model(inputs=[model_A.input,model_B.input], outputs=outputs, name="MySegNet_4")
def _build_model(self):
input_layer = self._input()
d0 = input_layer
d0 = self._conv_k3_activation_possible_batchnorm(64)(d0)
d0 = self._conv_k3_activation_possible_batchnorm(64)(d0)
d1 = self._max_pooling()(d0)
d1 = self._conv_k3_activation_possible_batchnorm(128)(d1)
d1 = self._conv_k3_activation_possible_batchnorm(128)(d1)
d2 = self._max_pooling()(d1)
d2 = self._conv_k3_activation_possible_batchnorm(256)(d2)
d2 = self._conv_k3_activation_possible_batchnorm(256)(d2)
d3 = self._max_pooling()(d2)
d3 = self._conv_k3_activation_possible_batchnorm(512)(d3)
d3 = self._conv_k3_activation_possible_batchnorm(512)(d3)
d3 = self._possible_dropout()(d3)
d4 = self._max_pooling()(d3)
d4 = self._conv_k3_activation_possible_batchnorm(1024)(d4)
d4 = self._conv_k3_activation_possible_batchnorm(1024)(d4)
d4 = self._possible_dropout()(d4)
u3 = self._trans_conv(512)(d4)
u3 = Concatenate()([d3, u3])
u3 = self._conv_k3_activation_possible_batchnorm(512)(u3)
u3 = self._conv_k3_activation_possible_batchnorm(512)(u3)
u2 = self._trans_conv(256)(u3)
u2 = Concatenate()([d2, u2])
u2 = self._conv_k3_activation_possible_batchnorm(256)(u2)
u2 = self._conv_k3_activation_possible_batchnorm(256)(u2)
u1 = self._trans_conv(128)(u2)
u1 = Concatenate()([d1, u1])
u1 = self._conv_k3_activation_possible_batchnorm(128)(u1)
u1 = self._conv_k3_activation_possible_batchnorm(128)(u1)
u0 = self._trans_conv(64)(u1)
u0 = Concatenate()([d0, u0])
u0 = self._conv_k3_activation_possible_batchnorm(64)(u0)
u0 = self._conv_k3_activation_possible_batchnorm(64)(u0)
difference_layer = self.get_difference_layer()(u0)
output_layer = Add(name=self._output_name)(
[input_layer, difference_layer])
if self._output_difference_layer:
model = Model(inputs=input_layer,
outputs=[output_layer, difference_layer],
name=self.name)
else:
model = Model(inputs=input_layer,
outputs=output_layer,
name=self.name)
self._model = model
self._input_layer = input_layer
self._output_layer = output_layer
self._difference_layer = difference_layer
def residual_block(X, filters):
shortcut = X
X = conv_block(X, filters, 5, 1)
X = Add()([shortcut, X])
X = common_layers(X)
return X
def get_train_model(config):
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception("Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# 输入进来的图片必须是2的6次方以上的倍数
input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]],
name="input_image")
# meta包含了一些必要信息
input_image_meta = Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
# RPN建议框网络的真实框信息
input_rpn_match = Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32)
input_rpn_bbox = Input(shape=[None, 4],
name="input_rpn_bbox",
dtype=tf.float32)
# 种类信息
input_gt_class_ids = Input(shape=[None],
name="input_gt_class_ids",
dtype=tf.int32)
# 框的位置信息
input_gt_boxes = Input(shape=[None, 4],
name="input_gt_boxes",
dtype=tf.float32)
# 标准化到0-1之间
gt_boxes = Lambda(lambda x: norm_boxes_graph(x,
K.shape(input_image)[1:3]))(
input_gt_boxes)
# mask语义分析信息
# [batch, height, width, MAX_GT_INSTANCES]
if config.USE_MINI_MASK:
input_gt_masks = Input(
shape=[config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
else:
input_gt_masks = Input(
shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
# 获得Resnet里的压缩程度不同的一些层
_, C2, C3, C4, C5 = get_resnet(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
# 组合成特征金字塔的结构
# P5长宽共压缩了5次
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5)
# P4长宽共压缩了4次
# Height/16,Width/16,256
P4 = Add(name="fpn_p4add")([
UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)
])
# P4长宽共压缩了3次
# Height/8,Width/8,256
P3 = Add(name="fpn_p3add")([
UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)
])
# P4长宽共压缩了2次
# Height/4,Width/4,256
P2 = Add(name="fpn_p2add")([
UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)
])
# 各自进行一次256通道的卷积，此时P2、P3、P4、P5通道数相同
# Height/4,Width/4,256
P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p2")(P2)
# Height/8,Width/8,256
P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p3")(P3)
# Height/16,Width/16,256
P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p4")(P4)
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p5")(P5)
# 在建议框网络里面还有一个P6用于获取建议框
# Height/64,Width/64,256
P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)
# P2, P3, P4, P5, P6可以用于获取建议框
rpn_feature_maps = [P2, P3, P4, P5, P6]
# P2, P3, P4, P5用于获取mask信息
mrcnn_feature_maps = [P2, P3, P4, P5]
anchors = get_anchors(config, config.IMAGE_SHAPE)
# 拓展anchors的shape，第一个维度拓展为batch_size
anchors = np.broadcast_to(anchors, (config.BATCH_SIZE, ) + anchors.shape)
# 将anchors转化成tensor的形式
anchors = Lambda(lambda x: tf.Variable(anchors),
name="anchors")(input_image)
# 建立RPN模型
rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
rpn_class_logits, rpn_class, rpn_bbox = [], [], []
# 获得RPN网络的预测结果，进行格式调整，把五个特征层的结果进行堆叠
for p in rpn_feature_maps:
logits, classes, bbox = rpn([p])
rpn_class_logits.append(logits)
rpn_class.append(classes)
rpn_bbox.append(bbox)
rpn_class_logits = Concatenate(axis=1,
name="rpn_class_logits")(rpn_class_logits)
rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class)
rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox)
# 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是
# rpn_class_logits : Batch_size, num_anchors, 2
# rpn_class : Batch_size, num_anchors, 2
# rpn_bbox : Batch_size, num_anchors, 4
proposal_count = config.POST_NMS_ROIS_TRAINING
# Batch_size, proposal_count, 4
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
active_class_ids = Lambda(lambda x: parse_image_meta_graph(x)[
"active_class_ids"])(input_image_meta)
if not config.USE_RPN_ROIS:
# 使用外部输入的建议框
input_rois = Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
name="input_roi",
dtype=np.int32)
# Normalize coordinates
target_rois = Lambda(
lambda x: norm_boxes_graph(x,
K.shape(input_image)[1:3]))(input_rois)
else:
# 利用预测到的建议框进行下一步的操作
target_rois = rpn_rois
"""找到建议框的ground_truth
Inputs:
proposals: [batch, N, (y1, x1, y2, x2)]建议框
gt_class_ids: [batch, MAX_GT_INSTANCES]每个真实框对应的类
gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]真实框的位置
gt_masks: [batch, height, width, MAX_GT_INSTANCES]真实框的语义分割情况
Returns:
rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]内部真实存在目标的建议框
target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]每个建议框对应的类
target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw)]每个建议框应该有的调整参数
target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width]每个建议框语义分割情况
"""
rois, target_class_ids, target_bbox, target_mask =\
DetectionTargetLayer(config, name="proposal_targets")([
target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])
# 找到合适的建议框的classifier预测结果
mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\
fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
# 找到合适的建议框的mask预测结果
mrcnn_mask = build_fpn_mask_graph(rois,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
output_rois = Lambda(lambda x: x * 1, name="output_rois")(rois)
# Losses
rpn_class_loss = Lambda(lambda x: rpn_class_loss_graph(*x),
name="rpn_class_loss")(
[input_rpn_match, rpn_class_logits])
rpn_bbox_loss = Lambda(lambda x: rpn_bbox_loss_graph(config, *x),
name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox])
class_loss = Lambda(lambda x: mrcnn_class_loss_graph(*x),
name="mrcnn_class_loss")([
target_class_ids, mrcnn_class_logits,
active_class_ids
])
bbox_loss = Lambda(lambda x: mrcnn_bbox_loss_graph(*x),
name="mrcnn_bbox_loss")(
[target_bbox, target_class_ids, mrcnn_bbox])
mask_loss = Lambda(lambda x: mrcnn_mask_loss_graph(*x),
name="mrcnn_mask_loss")(
[target_mask, target_class_ids, mrcnn_mask])
# Model
inputs = [
input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
input_gt_class_ids, input_gt_boxes, input_gt_masks
]
if not config.USE_RPN_ROIS:
inputs.append(input_rois)
outputs = [
rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits, mrcnn_class,
mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois, rpn_class_loss,
rpn_bbox_loss, class_loss, bbox_loss, mask_loss
]
model = Model(inputs, outputs, name='mask_rcnn')
return model
def create_darknet53(IMG_SIZE, num_categories=4):
inputs = Input(shape=(IMG_SIZE, IMG_SIZE, 3))
x = Block1(inputs, [3, 3], [1, 2], [32, 64])
x1 = Block1(x, [1, 3], [1, 1], [32, 64])
x = Add()([x, x1])
x = Block2(x, 3, 2, 128)
x1 = Block1(x, [1, 3], [1, 1], [64, 128])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [64, 128])
x = Add()([x, x1])
x = Block2(x, 3, 2, 256)
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [128, 256])
x = Add()([x, x1])
x = Block2(x, 3, 2, 512)
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [256, 512])
x = Add()([x, x1])
x = Block2(x, 3, 2, 1024)
x1 = Block1(x, [1, 3], [1, 1], [512, 1024])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [512, 1024])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [512, 1024])
x = Add()([x, x1])
x1 = Block1(x, [1, 3], [1, 1], [512, 1024])
x = Add()([x, x1])
x = AveragePooling2D()(x)
x = Flatten()(x)
outputs = Dense(num_categories, activation='softmax')(x)
model = Model(inputs, outputs)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
def Deep_Hedging_Model(N = None, d = None, m = None, \
risk_free = None, dt = None, initial_wealth = 0.0, epsilon = 0.0, \
final_period_cost = False, strategy_type = None, use_batch_norm = None, \
kernel_initializer = "he_uniform", \
activation_dense = "relu", activation_output = "linear",
delta_constraint = None, share_stretegy_across_time = False,
cost_structure = "proportional"):
# State variables.
prc = Input(shape=(1, ), name="prc_0")
information_set = Input(shape=(1, ), name="information_set_0")
inputs = [prc, information_set]
for j in range(N + 1):
if j < N:
# Define the inputs for the strategy layers here.
if strategy_type == "simple":
helper1 = information_set
elif strategy_type == "recurrent":
if j == 0:
# Tensorflow hack to deal with the dimension problem.
# Strategy at t = -1 should be 0.
# There is probably a better way but this works.
# Constant tensor doesn't work.
strategy = Lambda(lambda x: x * 0.0)(prc)
helper1 = Concatenate()([information_set, strategy])
# Determine if the strategy function depends on time t or not.
if not share_stretegy_across_time:
strategy_layer = Strategy_Layer(d = d, m = m,
use_batch_norm = use_batch_norm, \
kernel_initializer = kernel_initializer, \
activation_dense = activation_dense, \
activation_output = activation_output,
delta_constraint = delta_constraint, \
day = j)
else:
if j == 0:
# Strategy does not depend on t so there's only a single
# layer at t = 0
strategy_layer = Strategy_Layer(d = d, m = m,
use_batch_norm = use_batch_norm, \
kernel_initializer = kernel_initializer, \
activation_dense = activation_dense, \
activation_output = activation_output,
delta_constraint = delta_constraint, \
day = j)
strategyhelper = strategy_layer(helper1)
# strategy_-1 is set to 0
# delta_strategy = strategy_{t+1} - strategy_t
if j == 0:
delta_strategy = strategyhelper
else:
delta_strategy = Subtract(name="diff_strategy_" +
str(j))([strategyhelper, strategy])
if cost_structure == "proportional":
# Proportional transaction cost
absolutechanges = Lambda(lambda x: K.abs(x),
name="absolutechanges_" +
str(j))(delta_strategy)
costs = Dot(axes=1)([absolutechanges, prc])
costs = Lambda(lambda x: epsilon * x,
name="cost_" + str(j))(costs)
elif cost_structure == "constant":
# Tensorflow hack..
costs = Lambda(lambda x: epsilon + x * 0.0)(prc)
if j == 0:
wealth = Lambda(lambda x: initial_wealth - x,
name="costDot_" + str(j))(costs)
else:
wealth = Subtract(name="costDot_" + str(j))([wealth, costs])
# Wealth for the next period
# w_{t+1} = w_t + (strategy_t-strategy_{t+1})*prc_t
# = w_t - delta_strategy*prc_t
mult = Dot(axes=1)([delta_strategy, prc])
wealth = Subtract(name="wealth_" + str(j))([wealth, mult])
# Accumulate interest rate for next period.
FV_factor = np.exp(risk_free * dt)
wealth = Lambda(lambda x: x * FV_factor)(wealth)
prc = Input(shape=(1, ), name="prc_" + str(j + 1))
information_set = Input(shape=(1, ),
name="information_set_" + str(j + 1))
strategy = strategyhelper
if j != N - 1:
inputs += [prc, information_set]
else:
inputs += [prc]
else:
# The paper assumes no transaction costs for the final period
# when the position is liquidated.
if final_period_cost:
if cost_structure == "proportional":
# Proportional transaction cost
absolutechanges = Lambda(lambda x: K.abs(x),
name="absolutechanges_" +
str(j))(strategy)
costs = Dot(axes=1)([absolutechanges, prc])
costs = Lambda(lambda x: epsilon * x,
name="cost_" + str(j))(costs)
elif cost_structure == "constant":
# Tensorflow hack..
costs = Lambda(lambda x: epsilon + x * 0.0)(prc)
wealth = Subtract(name="costDot_" + str(j))([wealth, costs])
# Wealth for the final period
# -delta_strategy = strategy_t
mult = Dot(axes=1)([strategy, prc])
wealth = Add()([wealth, mult])
# Add the terminal payoff of any derivatives.
payoff = Input(shape=(1, ), name="payoff")
inputs += [payoff]
wealth = Add(name="wealth_" + str(j))([wealth, payoff])
return Model(inputs=inputs, outputs=wealth)
def create_model(opt,
metrics,
loss,
trainable_pretrained=True,
input_shape=(224, 224, 3)):
old_model = MobileNet(input_shape=input_shape,
weights='imagenet',
include_top=False)
old_model.trainable = trainable_pretrained
original_image = Lambda(
lambda x: x,
name='original_image',
# trainable=True
)(old_model.input)
x = old_model.output
y_names = [
"conv_pw_11_relu", "conv_pw_5_relu", "conv_pw_3_relu", "conv_pw_1_relu"
]
f_nums = [1024, 64, 64, 64]
ys = [
Conv2D(f_num, kernel_size=1, name=f'skip_hair_conv_{i}')(
old_model.get_layer(name=name).output)
for i, (name, f_num) in enumerate(zip(y_names, f_nums))
] + [None]
for i in range(5):
y = ys[i]
x = UpSampling2D(name=f'upsampling_hair_{i}')(x)
if y is not None:
x = Add(name=f'skip_hair_add_{i}')([x, y])
x = DepthwiseConv2D(
kernel_size=3,
padding='same',
name=f'depth_conv2d_hair_{i}',
kernel_initializer=GlorotNormal(seed=(i + 1)),
)(x)
x = Conv2D(
64,
kernel_size=1,
padding='same',
name=f'conv2d_hair_{i}',
kernel_regularizer=L2(2e-5),
kernel_initializer=GlorotNormal(seed=11 * (i + 1)),
)(x)
x = ReLU(name=f'relu_hair_{i}')(x)
x = Conv2D(
# 1,
2,
kernel_size=1,
padding='same',
name='conv2d_hair_final',
kernel_regularizer=L2(2e-5),
kernel_initializer=GlorotNormal(seed=0))(x)
x = Softmax(name='sigmoid_hair_final')(x)
x = Concatenate()([x, original_image])
# x = Activation('sigmoid', name='sigmoid_hair_final')(x)
model = Model(old_model.input, x)
if opt:
model.compile(
optimizer=opt,
loss=loss,
metrics=metrics,
)
return model
def _shuffle_unit(inputs,
in_channels,
out_channels,
groups,
bottleneck_ratio,
strides=2,
stage=1,
block=1):
"""
creates a shuffleunit
Parameters
----------
inputs:
Input tensor of with `channels_last` data format
in_channels:
number of input channels
out_channels:
number of output channels
strides:
An integer or tuple/list of 2 integers,
specifying the strides of the convolution along the width and height.
groups: int(1)
number of groups per channel
bottleneck_ratio: float
bottleneck ratio implies the ratio of bottleneck channels to output channels.
For example, bottleneck ratio = 1 : 4 means the output feature map is 4 times
the width of the bottleneck feature map.
stage: int(1)
stage number
block: int(1)
block number
Returns
-------
"""
if K.image_data_format() == 'channels_last':
bn_axis = -1
else:
bn_axis = 1
prefix = 'stage%d/block%d' % (stage, block)
#if strides >= 2:
#out_channels -= in_channels
# default: 1/4 of the output channel of a ShuffleNet Unit
bottleneck_channels = int(out_channels * bottleneck_ratio)
groups = (1 if stage == 2 and block == 1 else groups)
x = _group_conv(inputs,
in_channels,
out_channels=bottleneck_channels,
groups=(1 if stage == 2 and block == 1 else groups),
name='%s/1x1_gconv_1' % prefix)
x = CustomBatchNormalization(axis=bn_axis,
name='%s/bn_gconv_1' % prefix)(x)
x = Activation('relu', name='%s/relu_gconv_1' % prefix)(x)
x = Lambda(channel_shuffle,
arguments={'groups': groups},
name='%s/channel_shuffle' % prefix)(x)
x = DepthwiseConv2D(kernel_size=(3, 3),
padding="same",
use_bias=False,
strides=strides,
name='%s/1x1_dwconv_1' % prefix)(x)
x = CustomBatchNormalization(axis=bn_axis,
name='%s/bn_dwconv_1' % prefix)(x)
x = _group_conv(
x,
bottleneck_channels,
out_channels=out_channels if strides == 1 else out_channels -
in_channels,
groups=groups,
name='%s/1x1_gconv_2' % prefix)
x = CustomBatchNormalization(axis=bn_axis,
name='%s/bn_gconv_2' % prefix)(x)
if strides < 2:
ret = Add(name='%s/add' % prefix)([x, inputs])
else:
avg = AveragePooling2D(pool_size=3,
strides=2,
padding='same',
name='%s/avg_pool' % prefix)(inputs)
ret = Concatenate(bn_axis, name='%s/concat' % prefix)([x, avg])
ret = Activation('relu', name='%s/relu_out' % prefix)(ret)
return ret
def __init__(self,
input_dim=1,
output_dim=1,
exo_dim=0,
backcast_length=10,
forecast_length=1,
stack_types=(TREND_BLOCK, SEASONALITY_BLOCK),
nb_blocks_per_stack=3,
thetas_dim=(4, 8),
share_weights_in_stack=False,
hidden_layer_units=256,
nb_harmonics=None):
self.stack_types = stack_types
self.nb_blocks_per_stack = nb_blocks_per_stack
self.thetas_dim = thetas_dim
self.units = hidden_layer_units
self.share_weights_in_stack = share_weights_in_stack
self.backcast_length = backcast_length
self.forecast_length = forecast_length
self.input_dim = input_dim
self.output_dim = output_dim
self.exo_dim = exo_dim
self.input_shape = (self.backcast_length, self.input_dim)
self.exo_shape = (self.backcast_length, self.exo_dim)
self.output_shape = (self.forecast_length, self.output_dim)
self.weights = {}
self.nb_harmonics = nb_harmonics
assert len(self.stack_types) == len(self.thetas_dim)
x = Input(shape=self.input_shape, name='input_variable')
x_ = {}
for k in range(self.input_dim):
x_[k] = Lambda(lambda z: z[..., k])(x)
e_ = {}
if self.has_exog():
e = Input(shape=self.exo_shape, name='exos_variables')
for k in range(self.exo_dim):
e_[k] = Lambda(lambda z: z[..., k])(e)
else:
e = None
y_ = {}
for stack_id in range(len(self.stack_types)):
stack_type = self.stack_types[stack_id]
nb_poly = self.thetas_dim[stack_id]
for block_id in range(self.nb_blocks_per_stack):
backcast, forecast = self.create_block(x_, e_, stack_id,
block_id, stack_type,
nb_poly)
for k in range(self.input_dim):
x_[k] = Subtract()([x_[k], backcast[k]])
if stack_id == 0 and block_id == 0:
y_[k] = forecast[k]
else:
y_[k] = Add()([y_[k], forecast[k]])
for k in range(self.input_dim):
y_[k] = Reshape(target_shape=(self.forecast_length, 1))(y_[k])
x_[k] = Reshape(target_shape=(self.backcast_length, 1))(x_[k])
if self.input_dim > 1:
y_ = Concatenate()([y_[ll] for ll in range(self.input_dim)])
x_ = Concatenate()([x_[ll] for ll in range(self.input_dim)])
else:
y_ = y_[0]
x_ = x_[0]
if self.input_dim != self.output_dim:
y_ = Dense(self.output_dim, activation='linear', name='reg_y')(y_)
x_ = Dense(self.output_dim, activation='linear', name='reg_x')(x_)
inputs_x = [x, e] if self.has_exog() else x
n_beats_forecast = Model(inputs_x, y_, name=self._FORECAST)
n_beats_backcast = Model(inputs_x, x_, name=self._BACKCAST)
self.models = {
model.name: model
for model in [n_beats_backcast, n_beats_forecast]
}
self.cast_type = self._FORECAST
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
input_layer = Input(shape=(416, 416, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size *
4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
if len(out_index) == 0: out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(
count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
def f(net):
# format of conv_params:
# [ [nb_col="kernel width", nb_row="kernel height",
# subsample="(stride_vertical,stride_horizontal)",
# border_mode="same" or "valid"] ]
# B(3,3): orignal <<basic>> block
if direction == 'up':
conv_params = [[3, 3, (1, 1), "same"], [3, 3, (1, 1), "same"]]
else:
conv_params = [[3, 3, stride, "same"], [3, 3, (1, 1), "same"]]
n_bottleneck_plane = n_output_plane
# Residual block
for i, v in enumerate(conv_params):
if i == 0:
if n_input_plane != n_output_plane:
net = BatchNormalization(axis=CHANNEL_AXIS)(net)
net = Activation("relu")(net)
convs = net
else:
convs = BatchNormalization(axis=CHANNEL_AXIS)(net)
convs = Activation("relu")(convs)
convs = Conv2D(n_bottleneck_plane, (v[0], v[1]),
strides=v[2],
padding=v[3],
kernel_initializer=WEIGHT_INIT,
kernel_regularizer=l2(WEIGHT_DECAY),
use_bias=USE_BIAS)(convs)
if direction == 'up':
convs = UpSampling2D(stride)(convs)
else:
convs = BatchNormalization(axis=CHANNEL_AXIS)(convs)
convs = Activation("relu")(convs)
if dropout_probability > 0:
convs = Dropout(dropout_probability)(convs)
convs = Conv2D(n_bottleneck_plane, (v[0], v[1]),
strides=v[2],
padding=v[3],
kernel_initializer=WEIGHT_INIT,
kernel_regularizer=l2(WEIGHT_DECAY),
use_bias=USE_BIAS)(convs)
# Shortcut Conntection: identity function or 1x1 convolutional
# (depends on difference between input & output shape - this
# corresponds to whether we are using the first block in each
# group; see _layer() ).
if n_input_plane != n_output_plane:
shortcut_stride = 1 if direction == 'up' else stride
shortcut = Conv2D(n_output_plane, (1, 1),
strides=shortcut_stride,
padding="same",
kernel_initializer=WEIGHT_INIT,
kernel_regularizer=l2(WEIGHT_DECAY),
use_bias=USE_BIAS)(net)
if direction == 'up':
shortcut = UpSampling2D(stride)(shortcut)
else:
if stride == 1:
shortcut = net
elif direction == 'up':
shortcut = UpSampling2D(stride)(net)
else:
shortcut = AveragePooling2D(stride)(net)
return Add()([convs, shortcut])
def darknet_residual(x, filters):
prev = x
x = darknet_conv(x, filters // 2, 1)
x = darknet_conv(x, filters, 3)
x = Add()([prev, x])
return x
def CNNResBlockModel(config):
def regularization(lamda):
if config.regularization_method == 'L2':
return keras.regularizers.l2(lamda)
elif config.regularization_method == 'L1':
return keras.regularizers.l1(lamda)
else:
raise Exception('Use Only L2 / L1 regularization')
def activation(activation_name, x):
if activation_name == 'leaky_relu':
return LeakyReLU(alpha=config.alpha)(x)
else:
return Activation(activation_name)(x)
def highway_layer(value, gate_bias=-3):
# https://towardsdatascience.com/review-highway-networks-gating-function-to-highway-image-classification-5a33833797b5
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
dim = K.int_shape(value)[-1]
# gate_bias_initializer = tensorflow.keras.initializers.Constant(gate_bias)
# gate = Dense(units=dim, bias_initializer=gate_bias_initializer)(value)
# gate = Activation("sigmoid")(gate)
# TODO (just for yellow color...) NOTE: to keep dimensions matched, convolution gate instead of regular sigmoid
# gate (T in paper)
gate = Conv2D(size_list[i_hidden + config.CNN_ResBlock_conv_per_block -
1],
kernel_size=filt_list[-1],
padding='same',
activation='sigmoid',
bias_initializer=tensorflow.keras.initializers.Constant(
gate_bias))(value)
# negated (C in paper)
negated_gate = Lambda(lambda x: 1.0 - x,
output_shape=(size_list[-1], ))(gate)
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
transformed_gated = Multiply()([gate, transformed])
# UpSample value if needed
if value.shape.as_list()[-1] != negated_gate.shape.as_list()[-1]:
r = negated_gate.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
value = tf.keras.layers.UpSampling3D(size=(1, 1, int(r)))(value)
identity_gated = Multiply()([negated_gate, value])
value = Add()([transformed_gated, identity_gated])
return value
def skip_connection_layer(value):
nonlocal i_hidden
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
if value.shape.as_list()[-1] != transformed.shape.as_list()[-1]:
r = transformed.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
# apply convolution as transformation
value = Conv2D(size_list[i_hidden - 1],
kernel_size=filt_list[i_hidden - 1],
padding='same')(value)
value = Add()([value, transformed])
return value
def ResBlock(x):
for i in range(config.CNN_ResBlock_conv_per_block):
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
lamda_cnn = 0.0 if config.use_l2_in_cnn is False else lamda
x = Conv2D(size_list[i_hidden],
kernel_size=filt_list[i_hidden],
padding='same',
bias_regularizer=regularization(lamda_cnn),
kernel_regularizer=regularization(lamda_cnn),
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
i_hidden = i_hidden + 1
return x
def ResBlockLane(x):
nonlocal i_hidden
# ResBlocks
for i in range(len(config.CNN_ResBlock_highway)):
if config.CNN_ResBlock_highway[i] == "Highway":
x = highway_layer(value=x)
elif config.CNN_ResBlock_highway[i] == "Skip":
x = skip_connection_layer(value=x)
elif config.CNN_ResBlock_highway[i] == "None":
x = ResBlock(x=x)
else:
raise Exception('only Highway/Skip/None is allowed !')
# MaxPool and Dropout
if config.CNN_ResBlock_dropout[i] != 0:
x = Dropout(rate=config.CNN_ResBlock_dropout[i])(x)
x = MaxPooling2D(pool_size=pool_list[i])(x)
return x
global background_implicit_inference
# parameters
kernel_initalizer = config.kernel_initalizer # default is 'glorot_uniform'
lamda = config.Regularization_term
p_dropout = config.dropout
activation_name = config.activation
filt_dim2_list = config.Filter_shape_dim1 if config.Filter_shape_symmetric else config.Filter_shape_dim2
filt_list = [(x, y)
for x, y in zip(config.Filter_shape_dim1, filt_dim2_list)]
pool_list = [
(x, y) for x, y in zip(config.Pool_shape_dim1, config.Pool_shape_dim2)
]
size_list = config.hidden_size
dense_list = config.Dense_size
input_shape = config.model_input_dim
p_dropout_conv1d = config.CNN_ResBlock_dropout_conv1d
p_dropout_after_all_conv2d = config.dropout_after_all_conv2d
p_dropout_dense = config.Dense_dropout
i_hidden = 0
# Input Layer
input_layer = Input(shape=input_shape)
assert len(size_list) == len(filt_list)
assert len(pool_list) == len(config.CNN_ResBlock_highway) == len(
config.CNN_ResBlock_dropout)
assert config.CNN_ResBlock_conv_per_block * len(
config.CNN_ResBlock_highway) == len(size_list)
assert len(config.Conv1D_size) == len(config.Conv1D_kernel)
if config.ResBlockDouble:
x1 = input_layer
x1 = ResBlockLane(x1)
i_hidden = 0 # zero the hidden sizes counter
x2 = input_layer
x2 = ResBlockLane(x2)
x = Add()([x1, x2])
else:
x = input_layer
# ResBlocks
x = ResBlockLane(x)
# Flatten
x = Flatten()(x)
if p_dropout_after_all_conv2d != 0:
x = Dropout(rate=p_dropout_after_all_conv2d)(x)
# Conv1D
if len(config.Conv1D_size) != 0:
x = tf.expand_dims(x, axis=-1)
for i in range(len(config.Conv1D_size)):
x = Conv1D(filters=config.Conv1D_size[i],
kernel_size=config.Conv1D_kernel[i],
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
if p_dropout_conv1d[i] != 0.0:
x = Dropout(rate=p_dropout_conv1d[1])(x)
# post-Conv1D
if len(config.Conv1D_size) != 0:
x = MaxPooling1D(pool_size=config.Conv1D_pool)(x)
# x = BatchNormalization()(x)
x = Flatten()(x)
# Dense
for i in range(len(dense_list)):
x = Dense(dense_list[i],
kernel_regularizer=regularization(lamda),
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
if p_dropout_dense[i] != 0.0:
x = Dropout(rate=p_dropout_dense[i])(x)
# x = Dropout(rate=p_dropout)(x)
# x = BatchNormalization()(x)
if config.learn_background:
x = Dense(3, activation='softmax')(x)
else:
x = Dense(1, activation='sigmoid')(x)
output_layer = x
model = Model(input_layer, output_layer)
if config.learn_background:
if config.background_implicit_inference:
background_implicit_inference = True
model = BlockBackgroundModel(input_layer, output_layer)
# else:
# model = Model(input_layer, output_layer)
# model.summary()
return model
def _pep_block(inputs, proj_filters, filters, stride, expansion, block_id):
#in_channels = backend.int_shape(inputs)[-1]
in_channels = inputs.shape.as_list()[-1]
pointwise_conv_filters = int(filters)
x = inputs
prefix = 'pep_block_{}_'.format(block_id)
# Pre-project
x = Conv2D(proj_filters,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
name=prefix + 'preproject')(x)
x = CustomBatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'preproject_BN')(x)
x = ReLU(6., name=prefix + 'preproject_relu')(x)
# Expand
#x = Conv2D(int(expansion * in_channels), kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'expand')(x)
x = Conv2D(int(expansion * proj_filters),
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
name=prefix + 'expand')(x)
x = CustomBatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'expand_BN')(x)
x = ReLU(6., name=prefix + 'expand_relu')(x)
# Depthwise
if stride == 2:
x = ZeroPadding2D(padding=correct_pad(K, x, 3), name=prefix + 'pad')(x)
x = DepthwiseConv2D(kernel_size=3,
strides=stride,
activation=None,
use_bias=False,
padding='same' if stride == 1 else 'valid',
name=prefix + 'depthwise')(x)
x = CustomBatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'depthwise_BN')(x)
x = ReLU(6., name=prefix + 'depthwise_relu')(x)
# Project
x = Conv2D(pointwise_conv_filters,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
name=prefix + 'project')(x)
x = CustomBatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'project_BN')(x)
if in_channels == pointwise_conv_filters and stride == 1:
return Add(name=prefix + 'add')([inputs, x])
return x
def create_new_det(input_shape=(800, 700, 35),
kernel_regularizer=None,
downsample_factor=4,
reg_channels=8):
'''
This architecture key differences:
# Params: 3,969,031
'''
K.clear_session()
KERNEL_REG = kernel_regularizer
KERNEL_SIZE = {
"Layer1": 7,
"Block1": 3,
"Block2": 3,
"Block3": 3,
"Block4": 3,
"Header": 3,
"Out": 1
}
PADDING = 'same'
FILTERS = 256
CLASS_CHANNELS = 1
OUT_KERNEL_SIZE = 1
REGRESS_CHANNELS = reg_channels
BiFPN_filters = 128
inp = Input(shape=input_shape)
x = inp
with tf.name_scope("Backbone"):
with tf.name_scope("Block1"):
x = Conv2D(filters=FILTERS // 8,
kernel_size=KERNEL_SIZE['Layer1'],
padding=PADDING,
kernel_regularizer=KERNEL_REG,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = conv_block(x,
filters=FILTERS // 8,
kernel_size=KERNEL_SIZE['Block1'],
num_layers=1)
block1_out = x
# Downsize twice
block1_out = AveragePooling2D()(block1_out)
block1_out = create_sep_conv_block(block1_out,
FILTERS // 8,
KERNEL_SIZE['Block1'],
num_layers=1)
block1_out = AveragePooling2D()(block1_out)
block1_out = create_sep_conv_block(block1_out,
FILTERS // 8,
KERNEL_SIZE['Block1'],
num_layers=1)
# change number of channels from X -> BiFPN channels
block1_out = Conv2D(filters=BiFPN_filters,
kernel_size=1,
padding=PADDING,
use_bias=False)(block1_out)
block1_out = BatchNormalization()(block1_out)
block1_out = ReLU()(block1_out)
with tf.name_scope("Block2"):
#--------Downsampling--------#
x = MaxPooling2D()(x)
#--------Downsampling--------#
x = Conv2D(filters=FILTERS // 4,
kernel_size=KERNEL_SIZE['Block2'],
padding=PADDING,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = conv_block(x,
filters=FILTERS // 4,
kernel_size=KERNEL_SIZE['Block2'],
num_layers=1)
block2_out = x
# Downsize once
block2_out = AveragePooling2D()(block2_out)
block2_out = create_sep_conv_block(block2_out,
FILTERS // 4,
KERNEL_SIZE['Block1'],
num_layers=1)
# change number of channels from X -> BiFPN channels
block2_out = Conv2D(filters=BiFPN_filters,
kernel_size=1,
padding=PADDING,
use_bias=False)(block2_out)
block2_out = BatchNormalization()(block2_out)
block2_out = ReLU()(block2_out)
with tf.name_scope("Block3"):
#--------Downsampling--------#
x = MaxPooling2D()(x)
#--------Downsampling--------#
x = Conv2D(filters=FILTERS // 2,
kernel_size=KERNEL_SIZE['Block3'],
padding=PADDING,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = conv_block(x,
filters=FILTERS // 2,
kernel_size=KERNEL_SIZE['Block3'],
num_layers=2)
block3_out = x
# change number of channels from X -> BiFPN channels
block3_out = Conv2D(filters=BiFPN_filters,
kernel_size=1,
padding=PADDING,
use_bias=False)(block3_out)
block3_out = BatchNormalization()(block3_out)
block3_out = ReLU()(block3_out)
with tf.name_scope("Block4"):
#--------Downsampling--------#
x = MaxPooling2D()(x)
#--------Downsampling--------#
x = Conv2D(filters=FILTERS // 1,
kernel_size=KERNEL_SIZE['Block4'],
padding=PADDING,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = conv_block(x,
filters=FILTERS // 1,
kernel_size=KERNEL_SIZE['Block4'],
num_layers=5)
block4_out = x
# upsmaple once
block4_out = tf.image.resize(block4_out,
size=get_new_shape(block4_out, 2),
method=RESIZE_METHOD)
block4_out = SeparableConv2D(filters=FILTERS // 1,
kernel_size=KERNEL_SIZE['Block4'],
padding=PADDING)(block4_out)
# change number of channels from X -> BiFPN channels
block4_out = Conv2D(filters=BiFPN_filters,
kernel_size=1,
padding=PADDING,
use_bias=False)(block4_out)
block4_out = BatchNormalization()(block4_out)
block4_out = ReLU()(block4_out)
out1, out2, out3, out4 = build_BiFPN_v2(block1_out,
block2_out,
block3_out,
block4_out,
filters=BiFPN_filters)
out5, out6, out7, out8 = build_BiFPN_v2(block1_out,
block2_out,
block3_out,
block4_out,
filters=BiFPN_filters)
with tf.name_scope("FinalOutput"):
out1 = Add()([out1, out2, out3, out4])
out1 = create_conv_block(out1,
filters=BiFPN_filters,
kernel_size=3,
num_layers=1)
out2 = Add()([out5, out6, out7, out8])
out2 = create_conv_block(out2,
filters=BiFPN_filters,
kernel_size=3,
num_layers=1)
concat = Add()([out1, out2])
concat = create_conv_block(concat,
filters=BiFPN_filters,
kernel_size=3,
num_layers=1)
branch_1 = create_conv_block(concat,
filters=BiFPN_filters,
kernel_size=3,
num_layers=2)
branch_2 = create_conv_block(concat,
filters=BiFPN_filters,
kernel_size=3,
num_layers=2)
obj = Conv2D(CLASS_CHANNELS,
kernel_size=OUT_KERNEL_SIZE,
padding=PADDING,
activation='sigmoid',
name='objectness_map')(branch_1)
geo = Conv2D(REGRESS_CHANNELS,
kernel_size=OUT_KERNEL_SIZE,
padding=PADDING,
name='geometric_map')(branch_2)
out = Concatenate(axis=-1, name='output_map')([obj, geo])
model = Model(inp, out)
return model
def DarknetResidual(x, filters):
prev = x
x = DarknetConv(x, filters // 2, 1)
x = DarknetConv(x, filters, 3)
x = Add()([prev, x])
return x
def get_predict_model(config):
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception("Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# 输入进来的图片必须是2的6次方以上的倍数
input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]],
name="input_image")
# meta包含了一些必要信息
input_image_meta = Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
# 输入进来的先验框
input_anchors = Input(shape=[None, 4], name="input_anchors")
# 获得Resnet里的压缩程度不同的一些层
_, C2, C3, C4, C5 = get_resnet(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
# 组合成特征金字塔的结构
# P5长宽共压缩了5次
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5)
# P4长宽共压缩了4次
# Height/16,Width/16,256
P4 = Add(name="fpn_p4add")([
UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)
])
# P4长宽共压缩了3次
# Height/8,Width/8,256
P3 = Add(name="fpn_p3add")([
UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)
])
# P4长宽共压缩了2次
# Height/4,Width/4,256
P2 = Add(name="fpn_p2add")([
UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)
])
# 各自进行一次256通道的卷积，此时P2、P3、P4、P5通道数相同
# Height/4,Width/4,256
P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p2")(P2)
# Height/8,Width/8,256
P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p3")(P3)
# Height/16,Width/16,256
P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p4")(P4)
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p5")(P5)
# 在建议框网络里面还有一个P6用于获取建议框
# Height/64,Width/64,256
P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)
# P2, P3, P4, P5, P6可以用于获取建议框
rpn_feature_maps = [P2, P3, P4, P5, P6]
# P2, P3, P4, P5用于获取mask信息
mrcnn_feature_maps = [P2, P3, P4, P5]
anchors = input_anchors
# 建立RPN模型
rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
rpn_class_logits, rpn_class, rpn_bbox = [], [], []
# 获得RPN网络的预测结果，进行格式调整，把五个特征层的结果进行堆叠
for p in rpn_feature_maps:
logits, classes, bbox = rpn([p])
rpn_class_logits.append(logits)
rpn_class.append(classes)
rpn_bbox.append(bbox)
rpn_class_logits = Concatenate(axis=1,
name="rpn_class_logits")(rpn_class_logits)
rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class)
rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox)
# 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是
# rpn_class_logits : Batch_size, num_anchors, 2
# rpn_class : Batch_size, num_anchors, 2
# rpn_bbox : Batch_size, num_anchors, 4
proposal_count = config.POST_NMS_ROIS_INFERENCE
# Batch_size, proposal_count, 4
# 对先验框进行解码
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
# 获得classifier的结果
mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\
fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
detections = DetectionLayer(config, name="mrcnn_detection")(
[rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta])
detection_boxes = Lambda(lambda x: x[..., :4])(detections)
# 获得mask的结果
mrcnn_mask = build_fpn_mask_graph(detection_boxes,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
# 作为输出
model = Model([input_image, input_image_meta, input_anchors], [
detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class,
rpn_bbox
],
name='mask_rcnn')
return model
def _convert(self, arc_seq, verbose=True):
out_filters, pool_layers = self.get_out_filters(self.model_space)
inp = get_layer(x=None, state=self.inputs)
# this is assuming all choices have the same out_filters
stem_conv = Operation('conv1d',
kernel_size=8,
filters=out_filters[0],
activation="linear")
x = self.res_layer(stem_conv,
self.wsf,
inp,
name="stem_conv",
add_conv1_under_pool=self.add_conv1_under_pool)
start_idx = 0
layers = []
for layer_id in range(len(self.model_space)):
if verbose:
print("start_idx=%i, layer id=%i, out_filters=%i x %i" %
(start_idx, layer_id, out_filters[layer_id], self.wsf))
count = arc_seq[start_idx]
this_layer = self.model_space[layer_id][count]
if verbose: print(this_layer)
if layer_id == 0:
x = self.res_layer(
this_layer,
self.wsf,
x,
name="L%i" % layer_id,
add_conv1_under_pool=self.add_conv1_under_pool)
else:
x = self.res_layer(
this_layer,
self.wsf,
layers[-1],
name="L%i" % layer_id,
add_conv1_under_pool=self.add_conv1_under_pool)
if layer_id > 0:
skip = arc_seq[start_idx + 1:start_idx + layer_id + 1]
skip_layers = [
layers[i] for i in range(len(layers)) if skip[i] == 1
]
if verbose: print("skip=%s" % skip)
if len(skip_layers):
skip_layers.append(x)
x = Add(name="L%i_resAdd" % layer_id)(skip_layers)
x = BatchNormalization(name="L%i_resBn" % layer_id)(x)
if self.dropout_rate != 0:
x = Dropout(self.dropout_rate,
name="L%i_dropout" % layer_id)(x)
layers.append(x)
if layer_id in pool_layers:
pooled_layers = []
for i, layer in enumerate(layers):
pooled_layers.append(
self.factorized_reduction_layer(
layer,
out_filters[layer_id + 1] * self.wsf,
name="pool_at_%i_from_%i" % (layer_id, i)))
if verbose: print("pooled@%i, %s" % (layer_id, pooled_layers))
layers = pooled_layers
start_idx += 1 + layer_id
if verbose: print('-' * 80)
# fully-connected layer
if self.flatten_mode == 'GAP':
x = GlobalAveragePooling1D()(x)
elif self.flatten_mode == 'Flatten':
x = Flatten()(x)
else:
raise Exception("Unknown flatten mode: %s" % self.flatten_mode)
if self.dropout_rate != 0:
x = Dropout(self.dropout_rate)(x)
x = Dense(units=self.fc_units, activation="relu")(x)
out = get_layer(x=x, state=self.outputs)
model = Model(inputs=inp, outputs=out)
return model
def inverted_block(self, x, strides=(1, 1), **metaparameters):
""" Construct an Inverted Residual Block
x : input to the block
strides : strides
n_filters : number of filters
alpha : width multiplier
expansion : multiplier for expanding number of filters
"""
n_filters = metaparameters['n_filters']
alpha = metaparameters['alpha']
if 'alpha' in metaparameters:
alpha = metaparameters['alpha']
else:
alpha = self.alpha
if 'expansion' in metaparameters:
expansion = metaparameters['expansion']
else:
expansion = self.expansion
del metaparameters['n_filters']
# Remember input
shortcut = x
# Apply the width filter to the number of feature maps for the pointwise convolution
filters = int(n_filters * alpha)
n_channels = int(x.shape[3])
# Dimensionality Expansion (non-first block)
if expansion > 1:
# 1x1 linear convolution
x = self.Conv2D(x,
expansion * n_channels, (1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = self.BatchNormalization(x)
x = self.ReLU(x)
# Strided convolution to match number of filters
if strides == (2, 2):
x = ZeroPadding2D(padding=((0, 1), (0, 1)))(x)
padding = 'valid'
else:
padding = 'same'
# Depthwise Convolution
x = self.DepthwiseConv2D(x, (3, 3),
strides,
padding=padding,
use_bias=False,
**metaparameters)
x = self.BatchNormalization(x)
x = self.ReLU(x)
# Linear Pointwise Convolution
x = self.Conv2D(x,
filters, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = self.BatchNormalization(x)
# Number of input filters matches the number of output filters
if n_channels == filters and strides == (1, 1):
x = Add()([shortcut, x])
return x
