def cnn_encode(embed_input):
ca1 = SeparableConv1D(filters=64,
kernel_size=1,
padding='same',
activation='relu',
name='conv1')
ca2 = SeparableConv1D(filters=64,
kernel_size=2,
padding='same',
activation='relu',
name='conv2')
ca3 = SeparableConv1D(filters=64,
kernel_size=3,
padding='same',
activation='relu',
name='conv3')
mp = GlobalMaxPooling1D()
da = Dense(200, activation='relu', name='encode')
x1 = ca1(embed_input)
x1 = mp(x1)
x2 = ca2(embed_input)
x2 = mp(x2)
x3 = ca3(embed_input)
x3 = mp(x3)
x = Concatenate()([x1, x2, x3])
return da(x)
def rcnn(embed_input, class_num):
ra = LSTM(200, activation='tanh', return_sequences=True)
ba = Bidirectional(ra, merge_mode='concat')
ca1 = SeparableConv1D(filters=64,
kernel_size=1,
padding='same',
activation='relu')
ca2 = SeparableConv1D(filters=64,
kernel_size=2,
padding='same',
activation='relu')
ca3 = SeparableConv1D(filters=64,
kernel_size=3,
padding='same',
activation='relu')
mp = GlobalMaxPooling1D()
da1 = Dense(200, activation='relu')
da2 = Dense(class_num, activation='softmax')
x = ba(embed_input)
x1 = ca1(x)
x1 = mp(x1)
x2 = ca2(x)
x2 = mp(x2)
x3 = ca3(x)
x3 = mp(x3)
x = Concatenate()([x1, x2, x3])
x = da1(x)
x = Dropout(0.2)(x)
return da2(x)
def cnn_1d(embed_input1, embed_input2):
ca1 = SeparableConv1D(filters=64, kernel_size=1, padding='same', activation='relu')
ca2 = SeparableConv1D(filters=64, kernel_size=2, padding='same', activation='relu')
ca3 = SeparableConv1D(filters=64, kernel_size=3, padding='same', activation='relu')
mp = GlobalMaxPooling1D()
da1 = Dense(200, activation='relu')
da2 = Dense(200, activation='relu')
da3 = Dense(1, activation='sigmoid')
x1 = ca1(embed_input1)
x1 = mp(x1)
x2 = ca2(embed_input1)
x2 = mp(x2)
x3 = ca3(embed_input1)
x3 = mp(x3)
x = Concatenate()([x1, x2, x3])
x = da1(x)
y1 = ca1(embed_input2)
y1 = mp(y1)
y2 = ca2(embed_input2)
y2 = mp(y2)
y3 = ca3(embed_input2)
y3 = mp(y3)
y = Concatenate()([y1, y2, y3])
y = da1(y)
diff = Lambda(lambda a: K.abs(a))(Subtract()([x, y]))
prod = Multiply()([x, y])
z = Concatenate()([x, y, diff, prod])
z = da2(z)
z = Dropout(0.2)(z)
return da3(z)
def sepcnn_model(blocks,
filters,
kernel_size,
dropout_rate,
pool_size,
input_shape,
use_pretrained_embedding=False,
is_embedding_trainable=False,
embedding_matrix=None):
model = Sequential()
if use_pretrained_embedding:
model.add(
Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
input_length=input_shape[0],
weights=[embedding_matrix],
trainable=is_embedding_trainable))
else:
model.add(
Embedding(input_dim=num_features,
output_dim=embedding_dim,
input_length=input_shape[0]))
for _ in range(blocks - 1):
model.add(Dropout(rate=dropout_rate))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(rate=dropout_rate))
model.add(Dense(1, activation='sigmoid'))
return model
def build_model(self):
input_layer = Input((self.max_sentence_length, ))
embedding_layer = Embedding(
self.input_dim,
self.embedding_dim,
input_length=self.max_sentence_length)(input_layer)
embedding_layer = Dropout(0.5)(embedding_layer)
# conv1
conv_layer1 = SeparableConv1D(self.feature_maps[0],
self.kernel_sizes[0],
activation='relu',
strides=1,
padding='same',
depth_multiplier=4)(embedding_layer)
max_pool_layer1 = GlobalMaxPooling1D()(conv_layer1)
dense_layer1 = Dense(self.hidden_dim)(max_pool_layer1)
dense_layer1 = Dropout(0.5)(dense_layer1)
# conv2
conv_layer2 = SeparableConv1D(self.feature_maps[1],
self.kernel_sizes[1],
activation='relu',
strides=1,
padding='same',
depth_multiplier=4)(embedding_layer)
max_pool_layer2 = GlobalMaxPooling1D()(conv_layer2)
dense_layer2 = Dense(self.hidden_dim)(max_pool_layer2)
dense_layer2 = Dropout(0.5)(dense_layer2)
# conv3
conv_layer3 = SeparableConv1D(self.feature_maps[2],
self.kernel_sizes[2],
activation='relu',
strides=1,
padding='same',
depth_multiplier=4)(embedding_layer)
max_pool_layer3 = GlobalMaxPooling1D()(conv_layer3)
dense_layer3 = Dense(self.hidden_dim)(max_pool_layer3)
dense_layer3 = Dropout(0.5)(dense_layer3)
# concatenate conv channels
concat = concatenate([dense_layer1, dense_layer2, dense_layer3])
#concat = Dropout(0.5)(concat)
concat = Activation('relu')(concat)
output_layer = Dense(self.output_dim, activation='sigmoid')(concat)
self.model = Model(inputs=input_layer, outputs=output_layer)
if self.loss == 'custom_recall_spec':
self.loss_name = 'custom_recall_spec'
custom_loss = binary_recall_specificity_loss(self.recall_weight)
self.loss = custom_loss
elif self.loss == 'combined_loss':
self.loss_name = 'combined_loss'
custom_loss = combined_loss(self.bce_weight, self.recall_weight)
self.loss = custom_loss
self.model.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=self.metrics)
def build_sepconv(Traniable_embedding, embedding_matrix, max_len, kmer_size,
metrics, classes_1, classes_2, classes_3, classes_4,
classes_5, classes_6):
inp = Input(shape=(max_len, ), dtype='uint16')
max_features = 4**kmer_size + 1
if Traniable_embedding == True:
main = Embedding(max_features, 128)(inp)
else:
#main = Embedding(max_features, vector_size, weights=[embedding_matrix],trainable=False)(inp)
main = embedding_matrix(inp)
main = SeparableConv1D(84, (5))(main)
main = Dropout(0.5)(main)
main = LeakyReLU()(main)
main = SeparableConv1D(58, (9))(main)
main = Dropout(0.5)(main)
main = LeakyReLU()(main)
main = SeparableConv1D(
180,
(13),
)(main)
main = Dropout(0.5)(main)
main = LeakyReLU()(main)
main = Dense(2800)(main)
main = Dropout(0.5)(main)
main = LeakyReLU()(main)
#main = Dense(2800)(main)
main = GlobalAveragePooling1D()(main)
out1 = Dense(classes_1, activation='softmax')(main)
out2 = Dense(classes_2, activation='softmax')(main)
out3 = Dense(classes_3, activation='softmax')(main)
out4 = Dense(classes_4, activation='softmax')(main)
out5 = Dense(classes_5, activation='softmax')(main)
if classes_6 != 0:
out6 = Dense(classes_6, activation='softmax')(main)
else:
pass
if metrics == 'f1':
metrics = f1
elif metrics == 'precision-recall':
metrics = [keras_metrics.precision(), keras_metrics.recall()]
else:
pass
model = Model(inputs=[inp], outputs=[out1, out2, out3, out4, out5])
optimizer = Adam(lr=0.001)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=[metrics])
return model
def nochannel_nodil(classes):
InputSignal = Input(shape=(250, 22))
ConvBlock1 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(InputSignal)
ConvBlock1 = BatchNormalization()(ConvBlock1)
ConvBlock1 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock1)
ConvBlock1 = BatchNormalization()(ConvBlock1)
for _ in range(0, 4):
ConvBlock1 = SeparableConv1D(filters=32, kernel_size=9, strides=2, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock1)
ConvBlock1 = BatchNormalization()(ConvBlock1)
ConvBlock2 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(InputSignal)
ConvBlock2 = BatchNormalization()(ConvBlock2)
ConvBlock2 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock2)
ConvBlock2 = BatchNormalization()(ConvBlock2)
for _ in range(0, 4):
ConvBlock2 = SeparableConv1D(filters=32, kernel_size=9, strides=2, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock2)
ConvBlock2 = BatchNormalization()(ConvBlock2)
ConvBlock3 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(InputSignal)
ConvBlock3 = BatchNormalization()(ConvBlock3)
ConvBlock3 = SeparableConv1D(filters=32, kernel_size=9, strides=1, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock3)
ConvBlock3 = BatchNormalization()(ConvBlock3)
for _ in range(0, 4):
ConvBlock3 = SeparableConv1D(filters=32, kernel_size=9, strides=2, padding='same', activation='relu',
dilation_rate=1, kernel_regularizer='l2')(ConvBlock3)
ConvBlock3 = BatchNormalization()(ConvBlock3)
Concat_Layer = keras.layers.concatenate([ConvBlock1, ConvBlock2, ConvBlock3])
Concat_Layer = Flatten()(Concat_Layer)
FinalOutput = Dense(1024, activation='relu', kernel_regularizer='l2')(Concat_Layer)
FinalOutput = BatchNormalization()(FinalOutput)
FinalOutput = Dropout(.5)(FinalOutput)
FinalOutput = Dense(512, activation='relu', kernel_regularizer='l2')(FinalOutput)
FinalOutput = BatchNormalization()(FinalOutput)
FinalOutput = Dropout(.5)(FinalOutput)
FinalOutput = Dense(256, activation='relu', kernel_regularizer='l2')(FinalOutput)
FinalOutput = BatchNormalization()(FinalOutput)
FinalOutput = Dropout(.5)(FinalOutput)
EventPrediction = Dense(64, activation='relu', kernel_regularizer='l2')(FinalOutput)
EventPrediction = BatchNormalization()(EventPrediction)
EventPrediction = Dropout(.5)(EventPrediction)
EventPrediction = Dense(classes, activation='softmax', kernel_regularizer='l2', name='event_prediction')(
EventPrediction)
CompleteModel = Model(inputs=(InputSignal), outputs=[EventPrediction])
opt = keras.optimizers.sgd(lr=1e-3, momentum=.9, nesterov=True)
CompleteModel.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
return CompleteModel
def build_model(input_length,
emb_input_dim,
emb_out_dim,
lstm_hidden_units,
num_cls,
embedding_matrix=None):
l_input = Input(shape=(input_length, ))
if embedding_matrix is None:
l_emb = Embedding(emb_input_dim, emb_out_dim)(l_input)
#l_emb = Embedding(emb_input_dim, emb_out_dim, mask_zero=True)(l_input)
else:
l_emb = Embedding(emb_input_dim,
emb_out_dim,
weights=[embedding_matrix],
trainable=True)(l_input)
# add bilstm layer
l_posemb = Position_Embedding()(l_emb)
l_posemb = Dropout(0.1)(l_posemb)
l_cnn = SeparableConv1D(128,
3,
use_bias=False,
activation='relu',
padding='same',
kernel_initializer='he_normal')(l_posemb)
l_cnn = SeparableConv1D(128,
3,
use_bias=False,
activation='relu',
padding='same',
kernel_initializer='he_normal')(l_cnn)
l_cnn = Conv1D(128,
3,
use_bias=False,
padding='same',
kernel_initializer='he_normal')(l_cnn)
# add attention layer
l_att = Attention(nb_head=64, size_per_head=16)([l_cnn] * 3)
print('l_att.shape:', l_att.shape)
# add dense layer
l_dense = Dense(num_cls, activation='softmax')(l_att)
model = Model(l_input, l_dense)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
model.summary()
return model
def sepa_conv_block(x, dim):
x1 = SeparableConv1D(dim,
kernel_size=1,
strides=1,
padding='same',
activation='relu')(x)
x1 = BatchNormalization()(x1)
x1 = SeparableConv1D(dim,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x1)
x1 = BatchNormalization()(x1)
return x1
def BuildModel(lr=0.001, decay=0.0):
brand_name = Input(shape=[1], name='brand_name')
category_name = Input(shape=[1], name='category_name')
item_condition_id = Input(shape=[1], name='item_condition_id')
item_description = Input(shape=(50, ), name='item_description')
name = Input(shape=(10, ), name='name')
shipping = Input(shape=[1], name='shipping')
em_ctg = Embedding(ctg_num + 1, 10)(category_name)
em_brand = Embedding(brand_num + 1, 10)(brand_name)
em_desc = Embedding(20000, 100)(item_description)
em_name = Embedding(20000, 20)(name)
x = SeparableConv1D(64, 3, activation='relu')(em_name)
x = GlobalAveragePooling1D()(x)
y = SeparableConv1D(64, 6, activation='relu')(em_desc)
y = SeparableConv1D(128, 6, activation='relu')(y)
y = layers.Dropout(0.3)(y)
y = BatchNormalization()(y)
y = GlobalAveragePooling1D()(y)
main = layers.concatenate([
layers.Flatten()(em_brand),
layers.Flatten()(em_ctg),
item_condition_id,
y,
x,
shipping,
])
dense = Dense(64, activation='relu')(main)
dense = Dense(32, activation='relu')(dense)
dense = Dense(1, activation='linear')(dense)
model = Model([
brand_name,
category_name,
item_condition_id,
item_description,
name,
shipping,
], dense)
optimizer = Adam(lr=lr, decay=decay)
model.compile(loss="mse", optimizer=optimizer)
return model
def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id):
channel_axis = -1
in_channels = K.int_shape(inputs)[channel_axis]
pointwise_conv_filters = int(filters * alpha)
pointwise_filters = _make_divisible(pointwise_conv_filters, 8)
x = inputs
prefix = 'block_{}_'.format(block_id)
if block_id:
# Expand
x = Conv1D(expansion * in_channels, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'expand')(x)
x = BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x)
x = ReLU(6., name=prefix + 'expand_relu')(x)
else:
prefix = 'expanded_conv_'
# Depthwise
# if stride == 2:
# x = ZeroPadding1D(padding=1, name=prefix + 'pad')(x)
x = SeparableConv1D(in_channels, kernel_size=3, strides=stride, activation=None, use_bias=False, padding='same', name=prefix + 'depthwise')(x)
x = BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x)
x = ReLU(6., name=prefix + 'depthwise_relu')(x)
# Project
x = Conv1D(pointwise_filters, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'project')(x)
x = BatchNormalization(axis=channel_axis, epsilon=1e-3,momentum=0.999, name=prefix + 'project_BN')(x)
if in_channels == pointwise_filters and stride == 1:
return add([inputs, x])
return x
def quartznet_12x1_15_39(input_dim, output_dim=29):
input_data = Input(name='the_input', shape=(None, input_dim))
x = input_data
x = sep1d_bn_relu(x, 128, 15, strides=2, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 21, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 21, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 128, 21, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 27, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 27, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 27, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 33, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 33, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 33, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 256, 39, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 512, 1, strides=1, dilation_rate=1, padding="same")
conv_final = SeparableConv1D(output_dim,
1,
padding="same",
dilation_rate=2)(x)
y_pred = Activation('softmax', name='softmax')(conv_final)
model = Model(inputs=input_data, outputs=y_pred)
model.output_length = lambda x: x / 2
print(model.summary())
return model
def test_separable_convolution(self):
N, C, H, W = 2, 3, 5, 5
x = np.random.rand(N, H, W, C).astype(np.float32, copy=False)
model = Sequential()
model.add(
SeparableConv2D(filters=10,
kernel_size=(1, 2),
strides=(1, 1),
padding='valid',
input_shape=(H, W, C),
data_format='channels_last',
depth_multiplier=4))
model.add(
MaxPooling2D((2, 2),
strides=(2, 2),
data_format='channels_last'))
model.compile(optimizer='sgd', loss='mse')
self._test_one_to_one_operator_keras(model, x)
x = np.random.rand(N, H, C).astype(np.float32, copy=False)
model = Sequential()
model.add(
SeparableConv1D(filters=10,
kernel_size=2,
strides=1,
padding='valid',
input_shape=(H, C),
data_format='channels_last'))
model.compile(optimizer='sgd', loss='mse')
self._test_one_to_one_operator_keras(model, x)
def quartznet_12x1_shallow(input_dim, output_dim=29):
# No bn and relu activations, so probably not a good model
input_data = Input(name='the_input', shape=(None, input_dim))
x = input_data
x = sep1d_bn_relu(x, 192, 15, strides=2, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 192, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 192, 15, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 192, 21, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 192, 21, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 384, 27, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 384, 27, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 384, 33, strides=1, dilation_rate=1, padding="same")
x = sep1d_bn_relu(x, 512, 1, strides=1, dilation_rate=1, padding="same")
conv_final = SeparableConv1D(output_dim,
1,
padding="same",
dilation_rate=2)(x)
y_pred = Activation('softmax', name='softmax')(conv_final)
model = Model(inputs=input_data, outputs=y_pred)
model.output_length = lambda x: x / 2
print(model.summary())
return model
def cnn(embed_input1, embed_input2, embed_input3):
ca1 = SeparableConv1D(filters=64,
kernel_size=1,
padding='same',
activation='relu',
name='conv1')
ca2 = SeparableConv1D(filters=64,
kernel_size=2,
padding='same',
activation='relu',
name='conv2')
ca3 = SeparableConv1D(filters=64,
kernel_size=3,
padding='same',
activation='relu',
name='conv3')
mp = GlobalMaxPooling1D()
da = Dense(200, activation='relu', name='encode')
norm = Lambda(lambda a: K.sum(K.square(a), axis=-1, keepdims=True))
x1 = ca1(embed_input1)
x1 = mp(x1)
x2 = ca2(embed_input1)
x2 = mp(x2)
x3 = ca3(embed_input1)
x3 = mp(x3)
x = Concatenate()([x1, x2, x3])
x = da(x)
y1 = ca1(embed_input2)
y1 = mp(y1)
y2 = ca2(embed_input2)
y2 = mp(y2)
y3 = ca3(embed_input2)
y3 = mp(y3)
y = Concatenate()([y1, y2, y3])
y = da(y)
z1 = ca1(embed_input3)
z1 = mp(z1)
z2 = ca2(embed_input3)
z2 = mp(z2)
z3 = ca3(embed_input3)
z3 = mp(z3)
z = Concatenate()([z1, z2, z3])
z = da(z)
pos = norm(Subtract()([x, y]))
neg = norm(Subtract()([x, z]))
return Subtract()([pos, neg])
def residual_layer(self, in_layer):
rl = SeparableConv1D(RESIDUAL_LAYER_PARAMETERS["filters"],
RESIDUAL_LAYER_PARAMETERS["strides"],
padding='same',
data_format='channels_first',
name="Policy_Head_Conv")(in_layer)
rl = BatchNormalization()(rl)
rl = Activation("relu")(rl)
rl = SeparableConv1D(RESIDUAL_LAYER_PARAMETERS["filters"],
RESIDUAL_LAYER_PARAMETERS["strides"],
padding='same',
data_format='channels_first',
name="Policy_Head_Conv")(rl)
rl = BatchNormalization()(rl)
rl = merge([rl, in_layer], mode='sum') # Skip connection
rl = Activation("relu")(rl)
return rl
def residual_block(inputs, factor, filters):
dilation = 2 ** factor
# Residual block
n1 = conv_block(inputs, filters, dilation)
residual = TimeDistributed(SeparableConv1D(1, kernel_size=1, padding='same'))(n1)
# print residual.shape
outputs = keras.layers.add([inputs, residual])
return outputs
def sepcnn_model(blocks, filters, kernel_size, dropout_rate, pool_size,
input_shape, vocab_size):
model = Sequential()
model.add(
Embedding(input_dim=vocab_size + 1,
output_dim=embedding_dim,
input_length=input_shape[0]))
for _ in range(blocks - 1):
model.add(Dropout(rate=dropout_rate))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(rate=dropout_rate))
model.add(Dense(1, activation='sigmoid'))
model.summary()
return model
def main(args):
args = parse_train_args(args)
X_train = np.load(args.X_train)
X_valid = np.load(args.X_validate)
y_train = np.load(args.y_train)
y_valid = np.load(args.y_validate)
model_save_path = args.model_save_path
def lr_schedule(epoch, lr):
if epoch > 50:
if epoch % 5 == 0:
return lr * 0.95
return lr
lr_callback = LearningRateScheduler(lr_schedule)
callbacks = [lr_callback, EarlyStopping(monitor='val_loss', patience=3)]
input_shape = X_train.shape[1:]
num_output_classes = y_train.shape[1]
input_layer = Input(shape=input_shape)
conv_1 = Conv1D(filters=40,
kernel_size=3,
padding='same',
activation='relu',
kernel_regularizer=l2(0.01))(input_layer)
pool_1 = MaxPooling1D(pool_size=(2))(conv_1)
conv_2 = SeparableConv1D(filters=40,
kernel_size=3,
padding='same',
activation='relu',
kernel_regularizer=l2(0.01))(pool_1)
bn_1 = BatchNormalization()(conv_2)
flatten = Flatten()(bn_1)
predictions = Dense(num_output_classes, activation='softmax')(flatten)
model = Model(input_layer, predictions)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
batch_size = 10000
epochs = 150
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_valid, y_valid),
callbacks=callbacks)
model.save(model_save_path)
def decoder(x, n_output_channels=80):
x = conv_block(x, ShapeChange=UpSampling1D)
x = conv_block(x, ShapeChange=UpSampling1D)
x = conv_block(x, ShapeChange=UpSampling1D)
# Use a depthwise convolution to return something with a single dimension
x = SeparableConv1D(filters=n_output_channels, kernel_size=16, padding='same')(x)
return x
def big_XCEPTION(input_shape, num_classes):
img_input = Input(input_shape)
x = Conv1D(32, (1), strides=(2), use_bias=False)(img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Conv1D(64, (1), use_bias=False)(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv1D(128, (1), strides=(2), padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv1D(128, (1), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv1_act')(x)
x = SeparableConv1D(128, (1), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling1D((3), strides=(2), padding='same')(x)
x = layers.add([x, residual])
residual = Conv1D(256, (1), strides=(2), padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv1D(256, (1), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = Dropout(0.5)(x)
x = SeparableConv1D(256, (1), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling1D((3), strides=(2), padding='same')(x)
x = layers.add([x, residual])
x = Conv1D(
num_classes,
(1),
# kernel_regularizer=regularization,
padding='same')(x)
x = GlobalAveragePooling1D()(x)
output = Activation('softmax', name='predictions')(x)
model = Model(img_input, output)
return model
def _pool(self, embed, insz, **kwargs):
"""Override the base method from parent to provide text pooling facility.
Here that is a stack of depthwise separable convolutions with interleaved max pooling followed by a global
avg-over-time pooling
:param dsz: Word embeddings dim size
:param kwargs:
:return: nothing
"""
filtsz = kwargs['filtsz']
blocks = kwargs.get('layers', 2)
pdrop = kwargs.get('dropout', 0.5)
cmotsz = kwargs['cmotsz']
poolsz = kwargs.get('poolsz', 2)
input_ = embed
for _ in range(blocks - 1):
drop1 = Dropout(rate=pdrop)(input_)
sep1 = SeparableConv1D(filters=cmotsz,
kernel_size=filtsz,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same')(drop1)
sep2 = SeparableConv1D(filters=cmotsz,
kernel_size=filtsz,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same')(sep1)
input_ = MaxPooling1D(pool_size=poolsz)(sep2)
sep3 = SeparableConv1D(filters=cmotsz * 2,
kernel_size=filtsz,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same')(input_)
global_average_pooling = GlobalAveragePooling1D()(sep3)
drop2 = Dropout(rate=pdrop)(global_average_pooling)
return drop2
def create_neural_network(self):
"""
Draft Network Architecture for testing purpose
Input: 16x10 size
Try with 3 hidden layers for now. Each layer has 120 units with a dropout rate at 0.25 and relu as the activation layer
Use Flatten to map to a one dimension output with size 19 (first 10 maps to the cards to pick, and the last 9 map to the board
to drop the card)
There should be two output layers: one for the card and one for the move
"""
state_input = Input(shape = (fe.get_feature_dim(self.features), 2 * gm.START_HANDS))
# Testing for Conv1d layer
x = SeparableConv1D(self.params["filters"], 5, padding='same', data_format='channels_first', name="Conv_kernal_5")(state_input)
x = BatchNormalization()(x)
x = Activation(self.params["activation"])(x)
x = MaxPooling1D(padding='same')(x)
for i in range(self.params["conv_layers"]):
x = SeparableConv1D(self.params["filters"], 3, padding='same', data_format='channels_first', name="Conv1D_{}".format(i+1))(x)
x = BatchNormalization()(x)
x = Activation(self.params["activation"])(x)
x = MaxPooling1D(padding='same')(x)
x = Flatten()(x)
for i in range(self.params["dense_layers"]):
x = Dense(self.params["units"], activation=self.params["activation"], name="Dense_{}".format(i+1))(x)
if self.params["dropout"] > 0:
x = Dropout(self.params["dropout"])(x)
x1 = Bias()(x)
x2 = Bias()(x)
card_output = Dense(2 * gm.START_HANDS, activation=self.params["output_activation"], name='card_output')(x1)
move_output = Dense(gm.BOARD_SIZE ** 2, activation=self.params["output_activation"], name='move_output')(x2)
network = Model(input=state_input, output=[card_output, move_output])
return network
def policy_head(self, in_layer):
ph = SeparableConv1D(POLICY_HEAD_PARAMETERS["filters"],
POLICY_HEAD_PARAMETERS["strides"],
padding='same',
data_format='channels_first',
name="Policy_Head_Conv")(in_layer)
ph = BatchNormalization()(ph)
ph = Activation("relu")(ph)
ph = Dense(POLICY_HEAD_PARAMETERS["plane_number"],
name="Policy_head_dense")(ph)
return ph
def build(self, input, neighbour=None):
input = super(ConvolutionInput, self).build(input)
input = input.content if hasattr(input, "content") else input
if input.shape.ndims == 4:
if self._type == "normal":
if (self._padding == "valid"
and (self._kernel[0] > input.shape.dims[1].value
or self._kernel[1] > input.shape.dims[2].value)):
self._padding = "same"
return Conv2D(self._features,
self._kernel,
strides=self._stride,
padding=self._padding,
activation=self._activation,
name=Node.get_name(self))(input)
elif self._type == "separable":
return SeparableConv2D(self._features,
self._kernel,
strides=self._stride,
padding=self._padding,
activation=self._activation,
name=Node.get_name(self))(input)
elif self._type == "depthwise":
return DepthwiseConv2D(self._kernel,
strides=self._stride,
padding=self._padding,
activation=self._activation,
name=Node.get_name(self))(input)
elif input.shape.ndims == 3:
if self._type == "normal":
if (self._padding == "valid"
and (self._kernel[0] > input.shape.dims[1].value
or self._kernel[1] > input.shape.dims[2].value)):
self._padding = "same"
return Conv1D(self._features,
self._kernel[0],
strides=self._stride[0],
padding=self._padding,
activation=self._activation,
name=Node.get_name(self))(input)
elif self._type == "separable":
return SeparableConv1D(self._features,
self._kernel[0],
strides=self._stride[0],
padding=self._padding,
activation=self._activation,
name=Node.get_name(self))(input)
return input
def value_head(self, in_layer):
vh = SeparableConv1D(VALUE_HEAD_PARAMETERS["filters"],
VALUE_HEAD_PARAMETERS["strides"],
padding='same',
data_format='channels_first',
name="Value_Head_Conv")(in_layer)
vh = BatchNormalization()(vh)
vh = Activation("relu")(vh)
vh = Dense(VALUE_HEAD_PARAMETERS["hidden_size"])(vh)
vh = Activation("relu")(vh)
vh = Dense(1)(vh)
vh = Activation("tanh")(vh)
return vh
def crnn(embed_input, class_num):
ca1 = SeparableConv1D(filters=64,
kernel_size=1,
padding='same',
activation='relu')
ca2 = SeparableConv1D(filters=64,
kernel_size=2,
padding='same',
activation='relu')
ca3 = SeparableConv1D(filters=64,
kernel_size=3,
padding='same',
activation='relu')
ra = LSTM(200, activation='tanh')
da = Dense(class_num, activation='softmax')
x1 = ca1(embed_input)
x2 = ca2(embed_input)
x3 = ca3(embed_input)
x = Concatenate()([x1, x2, x3])
x = ra(x)
x = Dropout(0.2)(x)
return da(x)
def separableconv_block(x, filters, kernel_size, strides, se, ratio, act,
name):
y = SeparableConv1D(filters=filters,
kernel_size=kernel_size,
padding='same',
strides=strides,
kernel_initializer='VarianceScaling',
name='{}_separableconv'.format(name))(x)
if se:
y = squeezeExcite(y, ratio, name='{}_se'.format(name))
y = BatchNormalization(name='{}_bn'.format(name))(y)
y = Activation(act, name='{}_act'.format(name))(y)
return y
def sep1d_bn_relu(x,
filters,
kernel_size,
strides=1,
dilation_rate=1,
padding="same"):
x = SeparableConv1D(filters,
kernel_size,
strides=strides,
dilation_rate=dilation_rate,
padding=padding)(x)
x = BatchNormalization()(x)
x = Swish()(x)
return x
def __init__(self,
filters,
kernel_size,
num_blocks,
num_convs,
num_heads,
initializer=None,
regularizer=None,
dropout=.1):
conv_layers = []
attention_layers = []
feedforward_layers = []
for i in range(num_blocks):
conv_layers.append([])
for j in range(num_convs):
conv_layers[i].append(
SeparableConv1D(filters,
7,
padding='same',
depthwise_initializer=initializer,
pointwise_initializer=initializer,
depthwise_regularizer=regularizer,
pointwise_regularizer=regularizer,
activation='relu',
bias_regularizer=regularizer,
activity_regularizer=regularizer))
attention_layers.append(
MultiHeadAttention(filters, num_heads, initializer,
regularizer, dropout))
feedforward_layers.append([
Conv1D(filters,
1,
activation='relu',
kernel_initializer=initializer,
kernel_regularizer=regularizer,
bias_regularizer=regularizer),
Conv1D(filters,
1,
activation='linear',
kernel_initializer=initializer,
kernel_regularizer=regularizer,
bias_regularizer=regularizer)
])
self.conv_layers = conv_layers
self.attention_layers = attention_layers
self.feedforward_layers = feedforward_layers
self.num_blocks = num_blocks
self.num_convs = num_convs
self.dropout = dropout
