def __init__(self, config, **kwargs):
super().__init__(**kwargs)
if config.hidden_size % config.num_attention_heads != 0:
raise ValueError(
"The hidden size (%d) is not a multiple of the number of attention "
"heads (%d)" % (config.hidden_size, config.num_attention_heads)
)
self.output_attentions = config.output_attentions
self.num_attention_heads = config.num_attention_heads
assert config.hidden_size % config.num_attention_heads == 0
self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
self.all_head_size = self.num_attention_heads * self.attention_head_size
self.query = tf.keras.layers.Dense(
self.all_head_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="query"
)
self.key = tf.keras.layers.Dense(
self.all_head_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="key"
)
self.value = tf.keras.layers.Dense(
self.all_head_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="value"
)
self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob)
def __init__(self,
attention_probs_dropout_prob, # attention Dropout比例
sequence_length, # 句子的长度
vocab_size, # 词表大小
hidden_size, # 编码维度
num_hidden_layers, # Transformer总层数
num_attention_heads, # Attention的头数
intermediate_size, # FeedForward的隐层维度
hidden_act, # FeedForward隐层的激活函数
max_position_embeddings, # 最大序列长度
hidden_dropout_prob=None, # Dropout比例
name=None, # 模型名称
**kwargs
):
self.sequence_length = sequence_length
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.attention_head_size = hidden_size // num_attention_heads
self.intermediate_size = intermediate_size
self.max_position_embeddings = max_position_embeddings
self.attention_dropout_rate = attention_probs_dropout_prob or 0
self.hidden_dropout_rate = hidden_dropout_prob or 0
self.hidden_act = hidden_act
self.name = name
self.built = False
self.embeddings_x = Embedding(input_dim=self.vocab_size, output_dim=self.hidden_size, mask_zero=True,
embeddings_initializer=initializers.TruncatedNormal(stddev=0.02),
name='Embedding-Token')
self.embeddings_s = Embedding(input_dim=2, output_dim=self.hidden_size,
embeddings_initializer=initializers.TruncatedNormal(stddev=0.02),
name='Embedding-Segment')
def create_tf_model(self, name):
no_hidden_layers = len(self.hidden_layers)
self.inp = Input(shape=(self.input_size, ))
for i in range(no_hidden_layers):
if (i == 0):
outp = Dense(self.hidden_layers[0],
activation='linear',
kernel_initializer=initializers.TruncatedNormal(
stddev=0.1),
bias_initializer=initializers.Constant(1))(
self.inp)
outp = Activation('relu')(outp)
else:
outp = Dense(self.hidden_layers[i],
activation='linear',
kernel_initializer=initializers.TruncatedNormal(
stddev=0.1),
bias_initializer=initializers.Constant(1))(outp)
outp = Activation('relu')(outp)
outp = Dropout(0.5)(outp, training=self.mc_dropout)
if (no_hidden_layers == 0):
outp = Dense(self.output_classes, activation='linear')(self.inp)
self.predictions = Activation('softmax')(outp)
else:
outp = Dense(self.output_classes, activation='linear')(outp)
self.predictions = Activation('softmax')(outp)
self.model = Model(self.inp, self.predictions, name=name + '_keras')
self.get_final_layer_model_output = K.function(
[self.model.layers[0].input], [self.model.layers[-3].output])
self.model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def build_model(wordsCount,embeddingDimension,embeddingMatrix,wordLimit):
model1 = Sequential()
model1.add(Embedding(wordsCount, embeddingDimension, weights=[embeddingMatrix], input_length=wordLimit))
model1.add(Dropout(0.1))
model1.add(Convolution1D(filters=32, kernel_size=3, padding='same', activation='relu'))
model1.add(Dropout(0.1))
# model1.add(Convolution1D(filters=filter, kernel_size=3, padding='same', activation='relu'))
# model1.add(Dropout(0.3))
model1.add(LSTM(256, activation=None, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=2),
dropout=0.1))
model1.summary()
model2 = Sequential()
model2.add(Embedding(wordsCount, embeddingDimension, weights=[embeddingMatrix], input_length=wordLimit))
model2.add(Dropout(0.1))
model2.add(Convolution1D(filters=32, kernel_size=5, padding='same', activation='relu'))
model2.add(Dropout(0.1))
# model2.add(Convolution1D(filters=16, kernel_size=5, padding='same', activation='relu'))
# model2.add(Dropout(0.3))
model2.add(LSTM(256, activation=None, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=2),
dropout=0.1))
model2.summary()
model = Sequential()
model.add(Merge([model1, model2], mode='concat'))
model.add(Dense(256, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=2)))
model.add(Dropout(0.1))
# model.add(Dense(128 // 2, kernel_initializer=weights))
# model.add(Dropout(0.3))
model.add(Dense(1, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=2), name='output'))
model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.01, clipvalue=1.0))
return model
def build(self, input_shape):
total_up_scale = 2**self.up_scale
trunc_init0 = initializers.TruncatedNormal()
trunc_init1 = initializers.TruncatedNormal(stddev=0.1)
self.features = Sequential()
for i in range(self.up_scale):
is_last = i == self.up_scale - 1
out_features = self.filters if is_last else self.constant_filters
kernel_init0 = trunc_init0 if is_last else 'glorot_uniform'
kernel_init1 = trunc_init1 if is_last else 'glorot_uniform'
self.features.add(
SameConv(filters=out_features,
kernel_size=1,
strides=1,
activation='relu',
kernel_initializer=kernel_init0,
kernel_regularizer=regularizers.l2(1e-3)))
self.features.add(
layers.Conv2DTranspose(
out_features,
kernel_size=total_up_scale,
strides=2,
padding='same',
kernel_initializer=kernel_init1,
kernel_regularizer=regularizers.l2(1e-3)))
super().build(input_shape)
def create_tf_model(self, name):
# self.model = Sequential()
no_hidden_layers = len(self.hidden_layers)
#
# for i in range(no_hidden_layers):
# if(i == 0):
# self.model.add(Dense(self.hidden_layers[0], input_dim = self.input_size, activation = 'relu'))
# else:
# self.model.add(Dense(self.hidden_layers[i], activation = 'relu'))
#
# if(no_hidden_layers == 0):
# self.model.add(Dense(self.output_classes, input_dim = self.input_size, activation = 'sigmoid'))
# else:
# self.model.add(Dense(self.output_classes, activation = 'sigmoid'))
#
self.inp = Input(shape=(self.input_size, ))
for i in range(no_hidden_layers):
if (i == 0):
outp = Dense(self.hidden_layers[0],
activation='linear',
kernel_initializer=initializers.TruncatedNormal(
stddev=0.1),
bias_initializer=initializers.Constant(1))(
self.inp)
#kernel_regularizer = regularizers.l2(0.01)
#, activity_regularizer = regularizers.l1(0.01)
#outp = Dense(self.hidden_layers[0], activation='linear')(self.inp)
#outp = BatchNormalization()(outp)
outp = Activation('relu')(outp)
else:
outp = Dense(self.hidden_layers[i],
activation='linear',
kernel_initializer=initializers.TruncatedNormal(
stddev=0.1),
bias_initializer=initializers.Constant(1))(outp)
#kernel_regularizer = regularizers.l2(0.01)
#, activity_regularizer = regularizers.l1(0.01)
#outp = Dense(self.hidden_layers[i], activation='linear')(outp)
#outp = BatchNormalization()(outp)
outp = Activation('relu')(outp)
outp = Dropout(0.5)(outp, training=self.mc_dropout)
if (no_hidden_layers == 0):
outp = Dense(self.output_classes, activation='linear')(self.inp)
self.predictions = Activation('softmax')(outp)
else:
outp = Dense(self.output_classes, activation='linear')(outp)
self.predictions = Activation('softmax')(outp)
#self.model = Model(self.inp, outp, name=name + '_keras')
self.model = Model(self.inp, self.predictions, name=name + '_keras')
print(self.model.layers[-3].output.shape)
print(self.model.layers[-2].output.shape)
self.get_final_layer_model_output = K.function(
[self.model.layers[0].input], [self.model.layers[-3].output])
#self.get_preds = K.function([self.model.layers[0].input], [self.predictions])
self.model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def network():
inputs = Input((80, 80, 4))
a = Input(shape=(ACTIONS, ))
y = Input(shape=(1, ), dtype=float)
conv1 = Conv2D(filters=32,
strides=4,
activation='relu',
padding='same',
use_bias=True,
kernel_size=[8, 8],
kernel_initializer=initializer.TruncatedNormal(stddev=0.01),
bias_initializer=initializer.Constant(value=0.01))(inputs)
maxpool1 = MaxPooling2D(pool_size=2, strides=2, padding='same')(conv1)
conv2 = Conv2D(filters=64,
strides=2,
activation='relu',
padding='same',
use_bias=True,
kernel_size=[4, 4],
kernel_initializer=initializer.TruncatedNormal(stddev=0.01),
bias_initializer=initializer.Constant(value=0.01))(maxpool1)
#maxpool2 = MaxPooling2D(pool_size=2, strides=2, padding='same')(conv2)
conv3 = Conv2D(filters=64,
strides=1,
activation='relu',
padding='same',
use_bias=True,
kernel_size=[1, 1],
kernel_initializer=initializer.TruncatedNormal(stddev=0.01),
bias_initializer=initializer.Constant(value=0.01))(conv2)
#maxpool3 = MaxPooling2D(pool_size=2, strides=2, padding='same')(conv3)
fci = Flatten()(conv3)
fc1 = Dense(512,
activation='relu',
use_bias=True,
kernel_initializer=initializer.TruncatedNormal(stddev=0.01),
bias_initializer=initializer.Constant(value=0.01))(fci)
fc2 = Dense(ACTIONS,
activation='linear',
use_bias=True,
kernel_initializer=initializer.TruncatedNormal(stddev=0.01),
bias_initializer=initializer.Constant(value=0.01))(fc1)
mask = Dot(axes=1)([fc2, a])
model = Model([inputs, a, y], fc2)
opt = Adam(lr=0.0001)
model.compile(optimizer=opt, loss=custom_loss(mask, y))
model.summary()
return model
def build(self, input_shape):
feature_shape = tuple(input_shape[-self.feature_rank:])
self.mu = self.add_weight(shape=feature_shape,
initializer=initializers.TruncatedNormal(
mean=0.0, stddev=0.5),
regularizer=None,
trainable=True,
name='mu')
self.log_sigma = self.add_weight(
shape=feature_shape,
initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.5),
regularizer=None,
trainable=True,
name='sigma')
super(GaussianStochasticLayer, self).build(input_shape)
def get_kernel_init(type, param=None, seed=None):
kernel_init = None
if type == 'glorot_uniform':
kernel_init = initializers.glorot_uniform(seed=seed)
elif type == 'VarianceScaling':
kernel_init = initializers.VarianceScaling(seed=seed)
elif type == 'RandomNormal':
if param is None:
param = 0.04
kernel_init = initializers.RandomNormal(mean=0.0,
stddev=param,
seed=seed)
elif type == 'TruncatedNormal':
if param is None:
param = 0.045 # Best for non-normalized coordinates
# param = 0.09 # "Best" for normalized coordinates
kernel_init = initializers.TruncatedNormal(mean=0.0,
stddev=param,
seed=seed)
elif type == 'RandomUniform':
if param is None:
param = 0.055 # Best for non-normalized coordinates
# param = ?? # "Best" for normalized coordinates
kernel_init = initializers.RandomUniform(minval=-param,
maxval=param,
seed=seed)
return kernel_init
def create_qvalue_network(self, state_dim, action_dim):
# tools
K_INIT = initializers.TruncatedNormal(mean=0.0, stddev=1e-3)
# for dropout
K.set_learning_phase(1)
# build network model
S = Input(shape=[state_dim])
w1 = Dense(self.HIDDEN1_UNITS,
activation='elu',
kernel_initializer=K_INIT)(S)
w2_d = Dropout(rate=0.1)(w1)
w2 = Dense(self.HIDDEN1_UNITS,
activation='linear',
kernel_initializer=K_INIT)(w2_d)
A = Input(shape=[action_dim])
a1 = Dense(self.HIDDEN2_UNITS,
activation='linear',
kernel_initializer=K_INIT)(A)
h1 = layers.concatenate([w2, a1])
h2_d = Dropout(rate=0.1)(h1)
h2 = Dense(self.HIDDEN2_UNITS,
activation='elu',
kernel_initializer=K_INIT)(h2_d)
V = Dense(1, activation='linear')(h2)
model = Model(inputs=[S, A], outputs=V)
adam = Adam(lr=self.lr)
model.compile(loss=self.BATCH_LOSS, optimizer=adam)
return model, A, S
def create_value_network(self, state_dim):
# tools
K_INIT = initializers.TruncatedNormal(mean=0.0, stddev=1e-3)
K_REG = regularizers.l2(1e-3)
# build network model
S = Input(shape=[state_dim])
w1 = Dense(self.HIDDEN1_UNITS,
activation='elu',
kernel_initializer=K_INIT,
kernel_regularizer=K_REG)(S)
w2 = Dense(self.HIDDEN1_UNITS,
activation='elu',
kernel_initializer=K_INIT,
kernel_regularizer=K_REG)(w1)
h2 = Dense(self.HIDDEN2_UNITS,
activation='elu',
kernel_initializer=K_INIT,
kernel_regularizer=K_REG)(w2)
V = Dense(1, activation='linear')(h2)
model = Model(inputs=S, outputs=V)
adam = Adam(lr=self.lr)
model.compile(loss=self.BATCH_LOSS, optimizer=adam)
return model, S
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
self.dense = tf.keras.layers.Dense(
config.hidden_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="dense"
)
self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")
self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
self.dense = tf.keras.layers.Dense(
config.hidden_size,
kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range),
activation="tanh",
name="dense",
)
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
self.dense = tf.keras.layers.Dense(
config.intermediate_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="dense"
)
if isinstance(config.hidden_act, str):
self.intermediate_act_fn = ACT2FN[config.hidden_act]
else:
self.intermediate_act_fn = config.hidden_act
def build(self, input_shape):
if self.beta > 0:
self.prior = K.sigmoid(
self.add_weight(shape=tuple(input_shape[-self.feature_rank:]),
initializer=initializers.TruncatedNormal(
mean=0.0, stddev=0.5, seed=2019),
regularizer=None,
trainable=True,
name='pior'))
super(BernoulliStochasticLayer, self).build(input_shape)
def build_model():
# 网络模型结构：3072-512-256-3
model = Sequential()
# kernel_regularizer=regularizers.l2(0.01)
# keras.initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None)
# initializers.random_normal
# #model.add(Dropout(0.8))
model.add(Dense(512, input_shape=(3072,), activation="relu",
kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None),
kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.5))
model.add(
Dense(256, activation="relu", kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None),
kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.5))
model.add(Dense(len(lb.classes_), activation="softmax",
kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None),
kernel_regularizer=regularizers.l2(0.01)))
return model
def conv_batch_lrelu(input_tensor, numfilter, dim, strides=1):
input_tensor = Conv2D(
numfilter, (dim, dim),
strides=strides,
padding='same',
kernel_regularizer=regularizers.l2(0.0005),
kernel_initializer=initializers.TruncatedNormal(stddev=0.1),
use_bias=False)(input_tensor)
input_tensor = BatchNormalization()(input_tensor)
return LeakyReLU(alpha=0.1)(input_tensor)
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
self.dense = tf.keras.layers.Dense(
config.hidden_size, kernel_initializer=initializers.TruncatedNormal(stddev=config.initializer_range), name="dense"
)
if isinstance(config.hidden_act, str):
self.transform_act_fn = ACT2FN[config.hidden_act]
else:
self.transform_act_fn = config.hidden_act
self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")
def create_temporaly_convolutional_model(max_input_window_size, num_segments, num_syn_types, filter_sizes_per_layer, num_filters_per_layer,
activation_function_per_layer, l2_regularization_per_layer,
strides_per_layer, dilation_rates_per_layer, initializer_per_layer):
# define input and flatten it
binary_input_mat = Input(shape=(max_input_window_size, num_segments * num_syn_types), name='input_layer')
for k in range(len(filter_sizes_per_layer)):
num_filters = num_filters_per_layer[k]
filter_size = filter_sizes_per_layer[k]
activation = activation_function_per_layer[k]
l2_reg = l2_regularization_per_layer[k]
stride = strides_per_layer[k]
dilation_rate = dilation_rates_per_layer[k]
initializer = initializer_per_layer[k]
initializer = initializers.TruncatedNormal(stddev=initializer)
first_layer_bias_initializer = initializers.Constant(value=0.1)
if k == 0:
x = Conv1D(num_filters, filter_size, activation=activation, bias_initializer=first_layer_bias_initializer, kernel_initializer=initializer,
kernel_regularizer=l2(l2_reg), strides=stride, dilation_rate=dilation_rate, padding='causal', name='layer_%d' %(k + 1))(binary_input_mat)
else:
x = Conv1D(num_filters, filter_size, activation=activation, kernel_initializer=initializer, kernel_regularizer=l2(l2_reg),
strides=stride, dilation_rate=dilation_rate, padding='causal', name='layer_%d' %(k + 1))(x)
#x = BatchNormalization(name='layer_%d_BN'%(k+1))(x)
output_spike_init_weights = initializers.TruncatedNormal(stddev=0.05)
output_spike_init_bias = initializers.Constant(value=-2.5)
output_soma_init = initializers.TruncatedNormal(stddev=0.05)
output_spike_predictions = Conv1D(1, 1, activation='sigmoid', kernel_initializer=output_spike_init_weights, bias_initializer=output_spike_init_bias,
kernel_regularizer=l2(1e-8), padding='causal', name='spikes')(x)
output_soma_voltage_pred = Conv1D(1, 1, activation='linear' , kernel_initializer=output_soma_init, kernel_regularizer=l2(1e-8), padding='causal', name='soma')(x)
temporaly_convolutional_network_model = Model(inputs=binary_input_mat, outputs=[output_spike_predictions, output_soma_voltage_pred])
optimizer_to_use = Nadam(lr=0.0003)
temporaly_convolutional_network_model.compile(optimizer=optimizer_to_use, loss=['binary_crossentropy','mse'], loss_weights=[1.0, 0.003])
temporaly_convolutional_network_model.summary()
return temporaly_convolutional_network_model
def conv_batch_lrelu(input_tensor, numfilter, dim, strides=1):
# https://github.com/guigzzz/Keras-Yolo-v2/blob/f61286371cdc2d470e0811234f552c70bbd5caba/yolo_layer_utils.py#L18
input_tensor = Conv2D(
numfilter, (dim, dim),
strides=strides,
padding='same',
kernel_regularizer=regularizers.l2(0.0005),
kernel_initializer=initializers.TruncatedNormal(stddev=0.1),
use_bias=False)(input_tensor)
input_tensor = BatchNormalization()(input_tensor)
return LeakyReLU(alpha=0.1)(input_tensor)
def build(self, input_shape):
"""Build shared word embedding layer """
with tf.name_scope("word_embeddings"):
# Create and initialize weights. The random normal initializer was chosen
# arbitrarily, and works well.
self.word_embeddings = self.add_weight(
"weight",
shape=[self.vocab_size, self.hidden_size],
initializer=initializers.TruncatedNormal(stddev=self.initializer_range),
)
super().build(input_shape)
def element_model(element, input_vector_length, n_hlayers, dense_units,
activation, dropout, l1, l2, use_bias):
"""
Args:
element: Element label.
input_vector_length: Length of feature vector.
n_hlayers: Number of hidden layers.
dense_units: Number of neurons per hidden layer.
activation: Activation function.
dropout: Dropout fraction.
l1: L1 weight regularization penalty(i.e. LASSO regression).
l2: L2 weight regularization penalty (i.e. ridge regression).
Returns:
model: Keras model of element, to be called in structure_model().
"""
input_layer = Input(shape=(input_vector_length, ),
name='{}_inputs'.format(element))
kernel_init = initializers.TruncatedNormal(mean=0.0,
stddev=0.05,
seed=None)
# first layer (input)
if dropout > 0:
h_layer = Dropout(dropout,
name='{}_dropout_{}'.format(element,
'visible'))(input_layer)
else:
h_layer = input_layer
for i in range(n_hlayers):
# stack hidden layers
h_layer = Dense(dense_units,
activation=activation,
use_bias=use_bias,
kernel_initializer=kernel_init,
name='{}_hidden_{}'.format(element, i))(h_layer)
if l1 > 0 or l2 > 0:
h_layer = ActivityRegularization(l1=l1,
l2=l2,
name='{}_reg_{}'.format(
element, i))(h_layer)
if dropout > 0:
h_layer = Dropout(dropout, name='{}_dropout_{}'.format(element,
i))(h_layer)
output_layer = Dense(1, use_bias=use_bias,
name='{}_Ei'.format(element))(h_layer)
model = Model(input_layer, output_layer, name='{}_network'.format(element))
return model
def build(self, input_shape):
assert len(input_shape[0]) >= 2
input_dim = input_shape[0][-1]
self.mu = self.add_weight(shape=(self.num_comp, input_dim),
initializer=initializers.TruncatedNormal(
mean=0.0, stddev=0.1),
name='mu')
self.s = self.add_weight(shape=(self.num_comp, ),
initializer='ones',
name='s',
constraint=constraints.non_neg())
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
self.vocab_size = config.vocab_size
self.hidden_size = config.hidden_size
self.initializer_range = config.initializer_range
self.position_embeddings = tf.keras.layers.Embedding(
config.max_position_embeddings,
config.hidden_size,
embeddings_initializer=initializers.TruncatedNormal(stddev=self.initializer_range),
name="position_embeddings",
)
self.token_type_embeddings = tf.keras.layers.Embedding(
config.type_vocab_size,
config.hidden_size,
embeddings_initializer=initializers.TruncatedNormal(stddev=self.initializer_range),
name="token_type_embeddings",
)
# self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
# any TensorFlow checkpoint file
self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")
self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
def build(self, input_shape): assert len(input_shape) >= 4, \ "The input Tensor shape should be [None, h_image, w_image, ..., input_num_capsule*input_dim_capsule]" self.batch_size = input_shape[0] self.h_image, self.w_image = int(input_shape[1]), int(input_shape[2]) self.input_dim_capsule = int(input_shape[-1]/self.input_num_capsule) # Transform matrix with shape [self.num_capsule, self.input_num_capsule, # self.h_capsule, int(self.input_dim_capsule/self.w_capsule)] assert (self.input_dim_capsule -1) % self.h_capsule == 0, \ "The input capsule pose dimensions could not be coerced to match the pose dimensions of the output." self.input_w_capsule = int((self.input_dim_capsule -1)/self.h_capsule) self.W = self.add_weight(shape=(self.h_image, self.w_image, self.input_num_capsule, self.num_capsule, \ self.input_w_capsule, self.w_capsule), initializer=initializers.TruncatedNormal(mean=0., stddev=1.), name='W') self.bias = self.add_weight(shape=(1, self.num_capsule, 1), initializer='ones', name='bias') self.built = True
def decode_network(l1, output_bits):
kernel_init = initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=None)
bias_init = initializers.Constant(value=0.1)
l1 = Reshape((8, 8, 2))(l1)
l1 = UpSampling2D(size=(2, 2))(l1) # nearest neighbour interpolation
l1 = Conv2D(64, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = BatchNormalization()(l1)
l1 = LeakyReLU(alpha=0.05)(l1)
l1 = UpSampling2D(size=(2, 2))(l1)
l1 = Conv2D(32, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = BatchNormalization()(l1)
l1 = LeakyReLU(alpha=0.05)(l1)
l1 = UpSampling2D(size=(2, 2))(l1)
l1 = Conv2D(16, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = BatchNormalization()(l1)
l1 = LeakyReLU(alpha=0.05)(l1)
# 4th transpose of convolutional layer !
l1 = UpSampling2D(size=(2, 2))(l1)
l2 = Conv2D(output_bits, (3, 3),
strides=1,
activation='tanh',
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
return l2
def _build(self):
'''
Builds SuperLearner. Details written in the paper below.
- The Relative Performance of Ensemble Methods with Deep Convolutional
Neural Networks for Image Classification (https://arxiv.org/abs/1704.01664)
- Super Learner (http://biostats.bepress.com/ucbbiostat/paper222/)
Returns:
Probabilities for each label by weighted sum of all models' scores
'''
init = initializers.TruncatedNormal(mean = 1/4)
x = Input(shape = (10, len(self.models)))
y = Conv1D(1, 1, kernel_initializer = init)(x)
y = Flatten()(y)
y = Activation('softmax')(y)
return Model(x, y, name = MODEL_NAME)
def encode_network(input_img):
kernel_init = initializers.TruncatedNormal(mean=0.0,
stddev=0.02,
seed=None)
bias_init = initializers.Constant(value=0)
l1 = Conv2D(8, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(input_img)
l1 = LeakyReLU(alpha=0.05)(l1)
l1 = AveragePooling2D(
(2, 2))(l1) # at this point the output is of shape 8*24*24
l1 = Conv2D(16, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = LeakyReLU(alpha=0.05)(l1)
l1 = AveragePooling2D((2, 2), padding='same')(l1)
l1 = Conv2D(32, (3, 3),
strides=1,
padding='same',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = LeakyReLU(alpha=0.05)(l1)
l1 = AveragePooling2D((2, 2), padding='same')(l1)
l1 = Flatten()(
l1
) # at this point the output is of shape 32*16*16 = 8192 and this is reduced to size = 1000
l1 = Dense(1000,
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
l1 = Dense(128,
kernel_initializer=kernel_init,
bias_initializer=bias_init,
use_bias=True)(l1)
return l1
def interpretConv(layers, vd):
assert 'filters' in vd and 'size' in vd and 'pad' in vd and 'activation' in vd
filters = vd['filters']
size = vd['size']
stride = vd['stride']
pad = 'valid' if vd['pad'] == 0 else 'same'
batch_normalize = bool(vd.get('batch_normalize', 0))
ops = [
Conv2D(filters,
size,
strides=stride,
padding=pad,
kernel_regularizer=regularizers.l2(0.0005),
kernel_initializer=initializers.TruncatedNormal(stddev=0.1),
use_bias=not batch_normalize)
]
if batch_normalize:
ops.append(BatchNormalization())
actname = vd['activation']
if actname == 'leaky':
ops.append(LeakyReLU(alpha=0.1))
elif actname == 'linear':
pass
else:
raise ValueError(
'interpretConv - encountered unknown activation function: {}'.
format(actname))
tensor = layers[-1]
for op in ops:
tensor = op(tensor)
return tensor, ops
def DenseNet(nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.0, dropout_rate=0, weight_decay=1e-4, classes=1000, weights_path=None):
'''Instantiate the DenseNet 121 architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(256, 256, 1), name='data')
else:
concat_axis = 1
img_input = Input(shape=(1, 256, 256), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 32
nb_layers = [4, 4, 4]
# Initial convolution
x = Conv2D(nb_filter, (3, 3), strides=(1, 1), padding='valid', use_bias=False, name= 'conv_1')(img_input)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn1')(x)
# x = Scale(axis=concat_axis, name='conv1_scale1')(x)
x = Activation('relu', name='relu1.1')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(nb_filter,(3, 3), strides=(1, 1), name='pool1')(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn2')(x)
# x = Scale(axis=concat_axis, name='conv1_scale2')(x)
x = Activation('relu', name='relu1.2')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx+2
x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
# Add transition_block
x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x)
# x = Scale(axis=concat_axis, name='conv'+str(final_stage)+'_blk_scale')(x)
x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x)
x = GlobalAveragePooling2D(name='pool'+str(final_stage))(x)
x = Dense(50, use_bias=True, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None))(x)
x = Activation('relu')(x)
x = Dropout(0.5)(x)
x = Dense(50, use_bias=True, kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None))(x)
x = Activation('relu')(x)
x = Dropout(0.5)(x)
x = Dense(classes, name='fc6')(x)
x = Activation('softmax', name='prob')(x)
model = Model(img_input, x, name='densenet')
if weights_path is not None:
model.load_weights(weights_path)
return model
