def create_model(self, input_shape=(1, 32, 32), dense_dim=1000, dy=10, nb_filters=[64, 128], kernel_size=(3, 3), pool_size=(2, 2),
dropout=0.5, bn=True, output_activation='softmax', opt='adam'):
"""
Create DRCN model: convnet model followed by conv. autoencoder
Args:
_input (Tensor) : input layer
dense_dim (int) : dimensionality of the final dense layers
dy (int) : output dimensionality
nb_filter (list) : list of #Conv2D filters
kernel_size (tuple) : Conv2D kernel size
pool_size (tuple) : MaxPool kernel size
dropout (float) : dropout rate
bn (boolean) : batch normalization mode
output_activation (string) : act. function for output layer
opt (string) : optimizer
"""
[d1, d2, c] = input_shape
if opt == 'adam':
opt = Adam(lr=3e-4)
elif opt == 'rmsprop':
opt = RMSprop(lr=1e-4)
_input = Input(shape=input_shape)
# Create ConvNet
self.create_convnet(_input, dense_dim=dense_dim, dy=dy, nb_filters=nb_filters,
kernel_size=kernel_size, pool_size=pool_size, dropout=dropout,
bn=bn, output_activation=output_activation, opt=opt)
# Create ConvAE, encoder functions are shared with ConvNet
_h = _input
# Reconstruct Conv2D layers
for i, nf in enumerate(nb_filters):
_h = self.enc_functions[i](_h)
_h = Activation('relu')(_h)
if i < 2:
_h = MaxPooling2D(pool_size=pool_size, padding='same')(_h)
[_, wflat, hflat, cflat] = _h.get_shape().as_list()
_h = Flatten()(_h)
# Dense layers
for i in range(len(nb_filters), len(self.enc_functions)):
_h = self.enc_functions[i](_h)
_h = Activation('relu')(_h)
# Decoder
_h = Dense(dense_dim)(_h)
_h = Activation('relu')(_h)
_xdec = Dense(wflat*hflat*cflat)(_h)
_xdec = Activation('relu')(_xdec)
_xdec = Reshape((wflat, hflat, nb_filters[-1]))(_xdec)
i = 0
for nf in reversed(nb_filters):
_xdec = Conv2D(nf, kernel_size, padding='same')(_xdec)
_xdec = Activation('relu')(_xdec)
if i > 0:
_xdec = UpSampling2D(size=pool_size)(_xdec)
i += 1
_xdec = Conv2D(c, kernel_size, padding='same', activation=clip_relu)(_xdec)
self.convae_model = Model(input=_input, output=_xdec)
self.convae_model.compile(loss='mse', optimizer=opt)
print(self.convae_model.summary())
activation='relu',
subsample_length=1))
#model.add(MaxPooling1D(pool_length=2)) # 池化窗口大小
'''model.add(Convolution1D(input_shape = (look_back,1),
nb_filter=64,
filter_length=2,
border_mode='valid',
activation='relu',
subsample_length=1))'''
model.add(MaxPooling1D(pool_length=2))
#model.add(Dropout(0.25))
#model.add(Dense(250)) # Dense就是常用的全连接层，250代表output的shape为(*,250)
model.add(Dropout(0.25))
model.add(Activation('relu')) # ReLU : y = max(0,x)
model.add(Dense(1))
model.add(Activation('linear')) # Linear : y = x
# Print whole structure of the model
print(model.summary())
# training the train data with n epoch
model.compile(loss="mse", optimizer="rmsprop") # adam
model.fit(np.atleast_3d(np.array(train_x)),
np.atleast_3d(train_y),
epochs=100,
batch_size=80,
verbose=1,
shuffle=False)
def create_network(**kwargs):
"""construct a convolutional neural network with Resnet-style skip connections.
Arguments are the same as with the default CNNPolicy network, except the default
number of layers is 20 plus a new n_skip parameter
Keword Arguments:
- input_dim: depth of features to be processed by first layer (no default)
- board: width of the go board to be processed (default 19)
- filters_per_layer: number of filters used on every layer (default 128)
- layers: number of convolutional steps (default 20)
- filter_width_K: (where K is between 1 and <layers>) width of filter on
layer K (default 3 except 1st layer which defaults to 5).
Must be odd.
- n_skip_K: (where K is as in filter_width_K) number of convolutional
layers to skip with the linear path starting at K. Only valid
at K >= 1. (Each layer defaults to 1)
Note that n_skip_1=s means that the next valid value of n_skip_* is 3
A diagram may help explain (numbers indicate layer):
1 2 3 4 5 6
I--C--B--R--C--B--R--C--M--B--R--C--B--R--C--B--R--C--M ... M --R--F--O
\__________________/ \___________________________/ \ ... /
[n_skip_1 = 2] [n_skip_3 = 3]
I - input
B - BatchNormalization
R - ReLU
C - Conv2D
F - Flatten
O - output
M - merge
The input is always passed through a Conv2D layer, the output of which
layer is counted as '1'. Each subsequent [R -- C] block is counted as
one 'layer'. The 'merge' layer isn't counted; hence if n_skip_1 is 2,
the next valid skip parameter is n_skip_3, which will start at the
output of the merge
"""
defaults = {
"board": 19,
"filters_per_layer": 128,
"layers": 20,
"filter_width_1": 5
}
# copy defaults, but override with anything in kwargs
params = defaults
params.update(kwargs)
# create the network using Keras' functional API,
# since this isn't 'Sequential'
model_input = Input(shape=(params["input_dim"], params["board"],
params["board"]))
# create first layer
convolution_path = Conv2D(
input_shape=(),
filters=params["filters_per_layer"],
kernel_size=(params["filter_width_1"], params["filter_width_1"]),
kernel_initializer='uniform',
activation='linear', # relu activations done inside resnet modules
padding='same',
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None)(model_input)
def add_resnet_unit(path, K, **params):
"""Add a resnet unit to path starting at layer 'K',
adding as many (ReLU + Conv2D) modules as specified by n_skip_K
Returns new path and next layer index, i.e. K + n_skip_K, in a tuple
"""
# loosely based on https://github.com/keunwoochoi/residual_block_keras
# see also # keras docs here:
# http://keras.io/getting-started/functional-api-guide/#all-models-are-callable-just-like-layers
block_input = path
# use n_skip_K if it is there, default to 1
skip_key = "n_skip_%d" % K
n_skip = params.get(skip_key, 1)
for i in range(n_skip):
layer = K + i
# add BatchNorm
path = BatchNormalization()(path)
# add ReLU
path = Activation('relu')(path)
# use filter_width_K if it is there, otherwise use 3
filter_key = "filter_width_%d" % layer
filter_width = params.get(filter_key, 3)
# add Conv2D
path = Conv2D(filters=params["filters_per_layer"],
kernel_size=(filter_width, filter_width),
kernel_initializer='uniform',
activation='linear',
padding='same',
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None)(path)
# Merge 'input layer' with the path
path = add([block_input, path])
return path, K + n_skip
# create all other layers
layer = 1
while layer < params['layers']:
convolution_path, layer = add_resnet_unit(convolution_path, layer,
**params)
if layer > params['layers']:
print("Due to skipping, ended with {} layers instead of {}".format(
layer, params['layers']))
# since each layer's activation was linear, need one more ReLu
convolution_path = Activation('relu')(convolution_path)
# the last layer maps each <filters_per_layer> featuer to a number
convolution_path = Conv2D(filters=1,
kernel_size=(1, 1),
kernel_initializer='uniform',
name="policy_conv_last",
padding='same',
activation="linear",
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None)(convolution_path)
# flatten output
network_output = Flatten()(convolution_path)
# add a bias to each board location
network_output = Bias()(network_output)
# softmax makes it into a probability distribution
network_output = Activation('softmax')(network_output)
return Model(inputs=[model_input], outputs=[network_output])
# it can learn timestep 0 (should fail)
model.fit(Xmask12, X0_onehot, nb_epoch=1, batch_size=batch_size)
score = model.evaluate(Xmask12, X0_onehot, batch_size=batch_size)
if score < uniform_score * 0.9:
raise Exception(
'Somehow learned to copy timestep 0 despite masking 1, score %f' %
score)
# Another testing approach, just initialize models and make sure that prepending zeros doesn't affect
# their output
print("About to compile the second model")
model2 = Sequential()
model2.add(Embedding(5, 4, mask_zero=True))
model2.add(TimeDistributedDense(4, 4))
model2.add(Activation('time_distributed_softmax'))
model2.add(LSTM(4, 4, return_sequences=True))
model2.add(Activation('tanh'))
model2.add(GRU(4, 4, activation='softmax', return_sequences=True))
model2.add(
SimpleDeepRNN(4, 4, depth=2, activation='relu', return_sequences=True))
model2.add(SimpleRNN(4, 4, activation='relu', return_sequences=True))
model2.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
theano_mode=theano.compile.mode.FAST_RUN)
print("Compiled model2")
X2 = np.random.random_integers(1, 4, size=(2, 5))
y2 = np.random.random((X2.shape[0], X2.shape[1], 4))
ref = model2.predict(X2)
Num_testdata = len(Valid_lists)
start_time = time.time()
print('model building...')
model = Sequential()
model.add(
Convolution1D(1, 35, border_mode='same', bias=False,
input_shape=(None, 1)))
model.add(Convolution1D(15, 35, border_mode='same', bias=False))
model.add(BatchNormalization(mode=2, axis=-1))
model.add(Convolution1D(15, 35, border_mode='same', bias=False))
model.add(BatchNormalization(mode=2, axis=-1))
model.add(Convolution1D(1, 35, border_mode='same', bias=False))
model.add(Activation('tanh'))
model.summary()
#pdb.set_trace()
model.compile(loss='mse', optimizer='adam')
with open('FCN_b2a.json', 'w') as f: # save the model
f.write(model.to_json())
checkpointer = ModelCheckpoint(filepath='FCN_b2a.hdf5',
verbose=1,
save_best_only=True,
mode='min')
print('training...')
g1 = data_generator(Train_lists, Train_Air_paths, shuffle="True")
g2 = valid_generator(Valid_lists, Train_Air_paths, shuffle="False")
def build_generator(self, is_deconv=True):
'''
Create U-net skip connection generator
encoder: C64-C128-C256-C512-C512-C512-C512-C512
Exceptions: BatchNorm not apply to first C64 layer
U-net decoder: CD512-CD1024-CD1024-C1024-C1024-C512-C256-C128
followed by Conv2D(output_dim) and Tanh (3 in general, 2 for colorization)
Ck - Conv2D-BatchNorm-LeakyReLU with k (4,4) Conv2D filters,
CDk - Conv2D-BatchNorm-Dropout-ReLU with k (4,4) Conv2D filters
Encoder downsample by factor of 2
Decoder upsample by factor of 2 (using UpSample2D or Deconv2D)
LeakyReLU in encoder with leaky slope of 0.2
'''
# ----------------------------------------------------
# Build encoder e.g C64-C128-C256-C512-C512-C512-C512-C512
# ----------------------------------------------------
switcher = {
16: ["C64", "C128", "C256", "C512"],
32: ["C64", "C128", "C256", "C512", "C512"],
64: ["C64", "C128", "C256", "C512", "C512", "C512"],
128: ["C64", "C128", "C256", "C512", "C512", "C512", "C512"],
256:
["C64", "C128", "C256", "C512", "C512", "C512", "C512", "C512"],
512: [
"C64", "C128", "C256", "C512", "C512", "C512", "C512", "C512",
"C512"
]
}
unet_encoder_filters = switcher.get(
min(self.img_rows, self.img_cols),
"Invalid img size") # ensure the last activation tensor the shape
self.logger_model.info(
'UNet generator encoder filters : {}'.format(unet_encoder_filters))
unet_input_layer = Input(shape=self.img_dim, name="UNet_input")
encoder = unet_input_layer
encoder_list = [encoder]
for encoder_i, encoder_filter in enumerate(unet_encoder_filters):
filters = int(encoder_filter[1:])
if encoder_i == 0:
encoder = self.conv_block(encoder,
filters,
encoder_index=encoder_i,
filter_name=encoder_filter,
is_BatchNorm=False)
else:
encoder = self.conv_block(encoder,
filters,
encoder_index=encoder_i,
filter_name=encoder_filter)
encoder_list.append(encoder)
# ------------------------------------------------------------
# Build decoder CD512-CD1024-CD1024-C1024-C1024-C512-C256-C128
# ------------------------------------------------------------
switcher = {
16: ["CD512", "C512", "C256", "C128"],
32: ["CD512", "CD1024", "C512", "C256", "C128"],
64: ["CD512", "CD1024", "C1024", "C512", "C256", "C128"],
128: ["CD512", "CD1024", "C1024", "C1024", "C512", "C256", "C128"],
256: [
"CD512", "CD1024", "CD1024", "C1024", "C1024", "C512", "C256",
"C128"
],
512: [
"CD512", "CD1024", "CD1024", "C1024", "C1024", "C1024", "C512",
"C256", "C128"
]
}
unet_decoder_filters = switcher.get(
min(self.img_rows, self.img_cols),
"Invalid img size") # ensure the last activation tensor the shape
self.logger_model.info(
'UNet generator decoder filters: {}'.format(unet_decoder_filters))
decoder = encoder_list[-1]
decoder_list = [decoder]
for decoder_i, decoder_filter in enumerate(unet_decoder_filters):
if decoder_filter[0] == 'C' and not decoder_filter[0:2] == "CD":
# Ck - without Dropout`
is_dropout = False
filters = int(decoder_filter[1:])
elif decoder_filter[0:2] == "CD":
is_dropout = True
filters = int(decoder_filter[2:])
if decoder_i == len(unet_decoder_filters) - 1 or decoder_i == 0:
is_concatenate = False
else:
is_concatenate = True
if is_deconv:
# build decoder using deconv_block
decoder = self.deconv_block(decoder,
encoder_list[-decoder_i - 2],
filters,
decoder_index=decoder_i,
filter_name=decoder_filter,
is_BatchNorm=True,
dropout=is_dropout,
concatenate=is_concatenate)
else:
decoder = self.up_conv_block(decoder,
encoder_list[-decoder_i - 2],
filters,
decoder_index=decoder_i,
filter_name=decoder_filter,
is_BatchNorm=True,
dropout=is_dropout,
concatenate=is_concatenate)
decoder_list.append(decoder)
print('decoder tensor {} is passed'.format(decoder))
# Last two layers Conv(output_dim), Tanh
decoder = Conv2D(3, (4, 4),
name='Decoder_' + str(decoder_i + 1) + '_Conv_final',
padding="same",
data_format=self._data_format)(decoder)
decoder_list.append(decoder)
decoder = Activation("tanh",
name='Decoder_' + str(decoder_i + 2) + '_' +
'Tanh')(decoder)
decoder_list.append(decoder)
unet_generator = Model(input=[unet_input_layer],
output=[decoder],
name='UNet_Generator')
self.logger_model.info('UNet generator summary:')
unet_generator.summary(print_fn=self.logger_model.info)
plot_model(unet_generator, to_file='UNet_generator.png')
return unet_generator
def get_3D_Recurrent_DenseUnet():
inputs = Input((img_rows, img_cols, depth, 1))
#list of number of filters per block
depth_cnn = [32, 64, 128, 256]
##start of encoder block
##encoder block1
conv11 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(inputs)
conv11 = BatchNormalization()(conv11)
conv11 = Activation('relu')(conv11)
conc11 = concatenate([inputs, conv11], axis=4)
conv12 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(conc11)
conv12 = BatchNormalization()(conv12)
conv12 = Activation('relu')(conv12)
conc12 = concatenate([inputs, conv12], axis=4)
perm = Permute((3,1,2,4))(conc12)
pool1 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool1 = Permute((2,3,1,4))(pool1)
pool1 = SpatialDropout3D(0.1)(pool1)
#encoder block2
conv21 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(pool1)
conv21 = BatchNormalization()(conv21)
conv21 = Activation('relu')(conv21)
conc21 = concatenate([pool1, conv21], axis=4)
conv22 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(conc21)
conv22 = BatchNormalization()(conv22)
conv22 = Activation('relu')(conv22)
conc22 = concatenate([pool1, conv22], axis=4)
perm = Permute((3,1,2,4))(conc22)
pool2 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool2 = Permute((2,3,1,4))(pool2)
pool2 = SpatialDropout3D(0.1)(pool2)
#encoder block3
conv31 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(pool2)
conv31 = BatchNormalization()(conv31)
conv31 = Activation('relu')(conv31)
conc31 = concatenate([pool2, conv31], axis=4)
conv32 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(conc31)
conv32 = BatchNormalization()(conv32)
conv32 = Activation('relu')(conv32)
conc32 = concatenate([pool2, conv32], axis=4)
perm = Permute((3,1,2,4))(conc32)
pool3 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool3 = SpatialDropout3D(0.1)(pool3)
##end of encoder block
#ConvLSTM block
x = BatchNormalization()(ConvLSTM2D(filters =depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(pool3))
x = BatchNormalization()(ConvLSTM2D(filters =depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(x))
x = BatchNormalization()(ConvLSTM2D(filters = depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(x))
# start of decoder block
# decoder block1
up1 = TimeDistributed(Conv2DTranspose(depth_cnn[2], (2, 2), strides=(2, 2), padding='same'))(x)
up1 = Permute((2,3,1,4))(up1)
up6 = concatenate([up1, conc32], axis=4)
conv61 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(up6)
conv61 = BatchNormalization()(conv61)
conv61 = Activation('relu')(conv61)
conc61 = concatenate([up6, conv61], axis=4)
conv62 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(conc61)
conv62 = BatchNormalization()(conv62)
conv62 = Activation('relu')(conv62)
conv62 = concatenate([up6, conv62], axis=4)
#decoder block2
up2 = Permute((3,1,2,4))(conv62)
up2 = TimeDistributed(Conv2DTranspose(depth_cnn[1], (2, 2), strides=(2, 2), padding='same'))(up2)
up2 = Permute((2,3,1,4))(up2)
up7 = concatenate([up2, conv22], axis=4)
conv71 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(up7)
conv71 = BatchNormalization()(conv71)
conv71 = Activation('relu')(conv71)
conc71 = concatenate([up7, conv71], axis=4)
conv72 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(conc71)
conv72 = BatchNormalization()(conv72)
conv72 = Activation('relu')(conv72)
conv72 = concatenate([up7, conv72], axis=4)
#decoder block3
up3 = Permute((3,1,2,4))(conv72)
up3 = TimeDistributed(Conv2DTranspose(depth_cnn[0], (2, 2), strides=(2, 2), padding='same'))(up3)
up3 = Permute((2,3,1,4))(up3)
up8 = concatenate([up3, conv12], axis=4)
conv81 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(up8)
conv81 = BatchNormalization()(conv81)
conv81 = Activation('relu')(conv81)
conc81 = concatenate([up8, conv81], axis=4)
conv82 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(conc81)
conv82 = BatchNormalization()(conv82)
conv82 = Activation('relu')(conv82)
conc82 = concatenate([up8, conv82], axis=4)
##end of decoder block
conv10 = Conv3D(1, (1, 1, 1), activation='sigmoid')(conc82)
model = Model(inputs=[inputs], outputs=[conv10])
model.compile(optimizer=Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.000000199), loss = loss_function, metrics=[dice_coef])
return model
def pretrained_finetune(weights_path, freezeAndStack):
model = Sequential()
if freezeAndStack == True:
model.trainLayersIndividually = 1
#conv-spatial batch norm - relu #1
model.add(ZeroPadding2D((2, 2), input_shape=(3, 64, 64)))
model.add(
Convolution2D(64,
5,
5,
subsample=(2, 2),
W_regularizer=WeightRegularizer(l1=1e-7, l2=1e-7)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv1"
#conv-spatial batch norm - relu #2
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, subsample=(1, 1)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv2"
#conv-spatial batch norm - relu #3
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, subsample=(2, 2)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
model.add(Dropout(0.25))
print "added conv3"
#conv-spatial batch norm - relu #4
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, subsample=(1, 1)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv4"
#conv-spatial batch norm - relu #5
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, subsample=(2, 2)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv5"
#conv-spatial batch norm - relu #6
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, subsample=(1, 1)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
model.add(Dropout(0.25))
print "added conv6"
#conv-spatial batch norm - relu #7
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, subsample=(2, 2)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv7"
#conv-spatial batch norm - relu #8
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, subsample=(1, 1)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv8"
#conv-spatial batch norm - relu #9
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(1024, 3, 3, subsample=(2, 2)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added conv9"
model.add(Dropout(0.25))
#Affine-spatial batch norm -relu #10
model.add(Flatten())
model.add(Dense(512, W_regularizer=WeightRegularizer(l1=1e-5, l2=1e-5)))
model.add(BatchNormalization(epsilon=1e-06, mode=0, axis=1, momentum=0.9))
model.add(Activation('relu'))
print "added affine!"
model.add(Dropout(0.5))
#affine layer w/ softmax activation added
model.add(
Dense(200,
activation='softmax',
W_regularizer=WeightRegularizer(l1=1e-5, l2=1e-5))
) #pretrained weights assume only 100 outputs, we need to train this layer from scratch
print "added final affine"
if freezeAndStack == True:
for layer in model.layers:
layer.trainable = False
model.layers[1].trainable = True
model.load_weights(weights_path)
return model
embedding_matrix = np.zeros((len(word_index) + 1, 200))
for word, i in tqdm(word_index.items()):
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
model = Sequential()
model.add(Embedding(len(word_index) + 1,
200,
weights=[embedding_matrix],
input_length=max_len,
trainable=False))
model.add(SpatialDropout1D(0.3))
model.add(GRU(100, dropout=0.3, recurrent_dropout=0.3))
model.add(Dense(1000, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(1000, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto')
model.fit(xtrain_pad, y=ytrain, batch_size=512, epochs=100, verbose=1, validation_data=(xvalid_pad, yvalid), callbacks=[earlystop])
predictions = model.predict(xvalid_pad)
predictions = np.round(predictions).astype(int)
print('GRU')
print ("f1_score :", np.round(f1_score(yvalid, predictions),5))
1)) #row - no.of epochs, col (gets appended) - no of pooling
Pool_Train_Loss = np.zeros(shape=(nb_epoch, 1))
x_pool_All = np.zeros(shape=(1))
Y_train = np_utils.to_categorical(y_train, nb_classes)
print('Training Model Without Acquisitions in Experiment', e)
model = Sequential()
model.add(
Convolution2D(32,
3,
3,
border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu')) #using relu activation function
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(128, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(128, 3, 3))
def pretrained(weights_path, freezeAndStack):
model = Sequential()
pretrained_weights = get_weight_dict_from_path(weights_path)
#conv-spatial batch norm - relu #1
model.add(ZeroPadding2D((2, 2), input_shape=(3, 64, 64)))
model.add(
Convolution2D(
64,
5,
5,
subsample=(2, 2),
weights=[pretrained_weights['W1'], pretrained_weights['b1']],
W_regularizer=WeightRegularizer(l1=1e-7, l2=1e-7)))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma1'],
pretrained_weights['beta1'],
pretrained_weights['running_mean1'],
pretrained_weights['running_var1']
]))
model.add(Activation('relu'))
print "added conv1"
#conv-spatial batch norm - relu #2
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
64,
3,
3,
subsample=(1, 1),
weights=[pretrained_weights['W2'], pretrained_weights['b2']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma2'],
pretrained_weights['beta2'],
pretrained_weights['running_mean2'],
pretrained_weights['running_var2']
]))
model.add(Activation('relu'))
print "added conv2"
#conv-spatial batch norm - relu #3
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
128,
3,
3,
subsample=(2, 2),
weights=[pretrained_weights['W3'], pretrained_weights['b3']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma3'],
pretrained_weights['beta3'],
pretrained_weights['running_mean3'],
pretrained_weights['running_var3']
]))
model.add(Activation('relu'))
print "added conv3"
model.add(Dropout(0.25))
#print "added dropout"
#conv-spatial batch norm - relu #4
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
128,
3,
3,
subsample=(1, 1),
weights=[pretrained_weights['W4'], pretrained_weights['b4']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma4'],
pretrained_weights['beta4'],
pretrained_weights['running_mean4'],
pretrained_weights['running_var4']
]))
model.add(Activation('relu'))
print "added conv4"
#conv-spatial batch norm - relu #5
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
256,
3,
3,
subsample=(2, 2),
weights=[pretrained_weights['W5'], pretrained_weights['b5']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma5'],
pretrained_weights['beta5'],
pretrained_weights['running_mean5'],
pretrained_weights['running_var5']
]))
model.add(Activation('relu'))
print "added conv5"
#conv-spatial batch norm - relu #6
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
256,
3,
3,
subsample=(1, 1),
weights=[pretrained_weights['W6'], pretrained_weights['b6']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma6'],
pretrained_weights['beta6'],
pretrained_weights['running_mean6'],
pretrained_weights['running_var6']
]))
model.add(Activation('relu'))
print "added conv6"
model.add(Dropout(0.25))
#print "added dropout"
#conv-spatial batch norm - relu #7
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
512,
3,
3,
subsample=(2, 2),
weights=[pretrained_weights['W7'], pretrained_weights['b7']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma7'],
pretrained_weights['beta7'],
pretrained_weights['running_mean7'],
pretrained_weights['running_var7']
]))
model.add(Activation('relu'))
print "added conv7"
#conv-spatial batch norm - relu #8
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
512,
3,
3,
subsample=(1, 1),
weights=[pretrained_weights['W8'], pretrained_weights['b8']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma8'],
pretrained_weights['beta8'],
pretrained_weights['running_mean8'],
pretrained_weights['running_var8']
]))
model.add(Activation('relu'))
print "added conv8"
#conv-spatial batch norm - relu #9
model.add(ZeroPadding2D((1, 1)))
model.add(
Convolution2D(
1024,
3,
3,
subsample=(2, 2),
weights=[pretrained_weights['W9'], pretrained_weights['b9']]))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma9'],
pretrained_weights['beta9'],
pretrained_weights['running_mean9'],
pretrained_weights['running_var9']
]))
model.add(Activation('relu'))
print "added conv9"
model.add(Dropout(0.50))
#print "added dropout"
#Affine-spatial batch norm -relu #10
model.add(Flatten())
model.add(
Dense(512,
weights=[
np.transpose(np.asarray(pretrained_weights['W10'])),
pretrained_weights['b10']
],
W_regularizer=WeightRegularizer(l1=1e-4, l2=1e-4)))
model.add(
BatchNormalization(epsilon=1e-06,
mode=0,
axis=1,
momentum=0.9,
weights=[
pretrained_weights['gamma10'],
pretrained_weights['beta10'],
pretrained_weights['running_mean10'],
pretrained_weights['running_var10']
]))
model.add(Activation('relu'))
print "added affine!"
model.add(Dropout(0.75))
#print "added dropout!"
#affine layer w/ softmax activation added
model.add(
Dense(200,
activation='softmax',
W_regularizer=WeightRegularizer(l1=1e-4, l2=1e-4))
) #pretrained weights assume only 100 outputs, we need to train this layer from scratch... meh
print "added final affine"
if freezeAndStack == True:
for layer in model.layers:
layer.trainable = False
model.layers[1].trainable = True
return model
'''
from keras.layers.core import Dense, Dropout, Activation
#指在深度学习网络的训练过程中，对于神经网络单元，按照一定的概率将其暂时从网络中丢弃。
#dropout是CNN中防止过拟合提高效果的一个大杀器
from keras.optimizers import SGD
from keras.datasets import mnist
import numpy
'''
第一步：选择模型
'''
model = Sequential()
'''
第二步：构建网络层
'''
model.add(Dense(500, input_shape=(784, ))) # 输入层，28*28=784
model.add(Activation('tanh')) # 激活函数是tanh
#每个神经元都有激活函数： linear，sigmoid，tanh，softmax，LeakyReLU和PReLU
model.add(Dropout(0.5)) # 采用50%的dropout,防止过拟合
model.add(Dense(500)) # 隐藏层节点500个
model.add(Activation('tanh'))
model.add(Dropout(0.5))
model.add(Dense(10)) # 输出结果是10个类别，所以维度是10
model.add(Activation('softmax')) # 最后一层用softmax作为激活函数
'''
第三步：编译
'''
'''
参数：
optimizer：指定模型训练的1优化器；
np.random.shuffle(mat)
pieces = np.vsplit(mat, 6)
(train_x, train_y) = prepare_batch(np.vstack(pieces[:4]))
(test_x, test_y) = prepare_batch(pieces[4])
(valid_x, valid_y) = prepare_batch(pieces[5])
# Use neural net on data
batch_size = 16
nb_epoch = 15
model = Sequential([
Dense(32, input_dim=train_x.shape[1]),
Activation(LeakyReLU()),
Dropout(0.5),
Dense(32),
Activation(LeakyReLU()),
Dropout(0.5),
Dense(1)
])
model.summary()
model.compile(loss='mean_squared_error',
optimizer=RMSprop(lr=0.0001),
metrics=['accuracy'])
history = model.fit(train_x,
train_y,
def conv_func(x):
x = Conv2D(nb_filter, (nb_row, nb_col), strides=stride,
padding='same')(x)
x = InstanceNormalization()(x)
x = Activation("relu")(x)
return x
def train(self, data, trgt, n_neurons=1, trn_info=None, fold=0):
print 'NeuralClassication train function'
if fold > trn_info.n_folds or fold < -1:
print 'Invalid Fold...'
return None
[data_preproc, trgt_preproc] = self.preprocess(data,trgt,
trn_info=trn_info,fold=fold)
# Check if the file exists
file_name = '%s/%s_%s_train_fold_%i_neurons_%i_model.h5'%(self.preproc_path,
self.trn_info.date,
self.name,fold,n_neurons)
if not os.path.exists(file_name):
best_init = 0
best_loss = 999
best_model = None
best_desc = {}
train_id, test_id = self.trn_info.CVO[fold]
for i_init in range(self.trn_info.n_inits):
print 'Init: %i of %i'%(i_init+1,self.trn_info.n_inits)
model = Sequential()
model.add(Dense(n_neurons, input_dim=data.shape[1], init="uniform"))
model.add(Activation('softplus'))
model.add(Dense(trgt.shape[1], init="uniform"))
model.add(Activation('softmax'))
adam = Adam(lr = self.trn_info.learning_rate,
# decay = self.trn_info.learning_decay,
beta_1 = self.trn_info.beta_1,
beta_2 = self.trn_info.beta_2,
epsilon = self.trn_info.epsilon)
model.compile(loss='mean_squared_error',
optimizer='Adam',
metrics=['accuracy'])
# Train model
earlyStopping = callbacks.EarlyStopping(monitor='val_loss',
patience=self.trn_info.patience,
verbose=self.trn_info.train_verbose,
mode='auto')
init_trn_desc = model.fit(data_preproc[train_id], trgt_preproc[train_id],
nb_epoch=self.trn_info.n_epochs,
batch_size=self.trn_info.batch_size,
callbacks=[earlyStopping],
verbose=self.trn_info.verbose,
validation_data=(data_preproc[test_id],
trgt_preproc[test_id]),
shuffle=True)
if np.min(init_trn_desc.history['val_loss']) < best_loss:
best_init = i_init
best_loss = np.min(init_trn_desc.history['val_loss'])
best_model = model
best_desc['epochs'] = init_trn_desc.epoch
best_desc['acc'] = init_trn_desc.history['acc']
best_desc['loss'] = init_trn_desc.history['loss']
best_desc['val_loss'] = init_trn_desc.history['val_loss']
best_desc['val_acc'] = init_trn_desc.history['val_acc']
# Save the model
file_name = '%s/%s_%s_train_fold_%i_neurons_%i_model.h5'%(self.train_path,
self.trn_info.date,
self.name,fold,n_neurons)
best_model.save(file_name)
# Save the descriptor
file_name = '%s/%s_%s_train_fold_%i_neurons_%i_trn_desc.jbl'%(self.train_path,
self.trn_info.date,
self.name,fold,n_neurons)
joblib.dump([best_desc],file_name,compress=9)
else:
# Load the model
file_name = '%s/%s_%s_train_fold_%i_neurons_%i_model.h5'%(self.train_path,
self.trn_info.date,
self.name,fold,n_neurons)
best_model = load_model(file_name)
# Load the descriptor
file_name = '%s/%s_%s_train_fold_%i_neurons_%i_trn_desc.jbl'%(self.train_path,
self.trn_info.date,
self.name,fold,n_neurons)
[best_desc] = joblib.load(file_name)
return [best_model, best_desc]
random_state=4)
uniques, id_train = np.unique(y_train, return_inverse=True)
Y_train = np_utils.to_categorical(id_train, nb_classes)
uniques, id_test = np.unique(y_test, return_inverse=True)
Y_test = np_utils.to_categorical(id_test, nb_classes)
print(Y_test)
model = Sequential()
model.add(Conv1D(nb_filters, 3, activation='relu', input_shape=(3, 5)))
model.add(Conv1D(nb_filters, 1, activation="relu"))
# model.add(Conv1D(16,1,activation="relu"))
model.add(Dense(128))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
Y_test = np.reshape(Y_test, (len(Y_test), 1, 15))
Y_train = np.reshape(Y_train, (len(Y_train), 1, 15))
# nb_epoch=5;
import operator
batch_size = 5
model.fit(x_train,
Y_train,
batch_size=batch_size,
nb_epoch=250,
verbose=1,
import numpy as np
from sklearn.datasets import load_iris
from keras.models import Sequential
from keras.layers.core import Dense,Activation
iris = load_iris()
data = iris.data
target = iris.target
data = data[target<2]
target = target[target<2]
model = Sequential()
# 输入层
model.add(Dense(10,input_dim=4))
model.add(Activation('relu'))
# 输出层
model.add(Dense(1,input_dim=1))
model.add(Activation('sigmoid'))
# 模型的编译
model.compile(loss='mean_squared_error',optimizer='adam')
# 训练
model.fit(data,target,nb_epoch=1000,batch_size=6)
# 预测
rst = model.predict_classes(data)
print(rst.reshape(len(target)))
print(target)
def run_cross_validation(nfolds=10):
# input image dimensions
img_rows, img_cols = 24, 32
batch_size = 64
nb_classes = 10
nb_epoch = 1
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
nb_pool = 2
# convolution kernel size
nb_conv = 3
random_state = 51
train_data, train_target = read_and_normalize_train_data(
img_rows, img_cols)
test_data, test_id = read_and_normalize_test_data(img_rows, img_cols)
yfull_train = dict()
yfull_test = []
kf = KFold(len(train_data),
n_folds=nfolds,
shuffle=True,
random_state=random_state)
num_fold = 0
for train_index, test_index in kf:
num_fold += 1
print('Start KFold number {} from {}'.format(num_fold, nfolds))
X_train, X_valid = train_data[train_index], train_data[test_index]
Y_train, Y_valid = train_target[train_index], train_target[test_index]
print('Split train: ', len(X_train))
print('Split valid: ', len(X_valid))
model = Sequential()
model.add(
Convolution2D(nb_filters,
nb_conv,
nb_conv,
border_mode='valid',
input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adadelta')
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
verbose=1,
validation_data=(X_valid, Y_valid))
# score = model.evaluate(X_valid, Y_valid, show_accuracy=True, verbose=0)
# print('Score log_loss: ', score[0])
predictions_valid = model.predict(X_valid, batch_size=128, verbose=1)
score = log_loss(Y_valid, predictions_valid)
print('Score log_loss: ', score)
# Store valid predictions
for i in range(len(test_index)):
yfull_train[test_index[i]] = predictions_valid[i]
# Store test predictions
test_prediction = model.predict(test_data, batch_size=128, verbose=1)
yfull_test.append(test_prediction)
score = log_loss(train_target, dict_to_list(yfull_train))
print('Final score log_loss: ', score)
test_res = merge_several_folds_fast(yfull_test, nfolds)
create_submission(test_res, test_id, score)
# Split the dataset
X_train, X_val, y_train, y_val = train_test_split(x,
y,
test_size=0.2,
random_state=10)
# In[6]:
#%%
# Defining the model
input_shape = img_data[0].shape
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same', input_shape=input_shape))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
#model.add(Convolution2D(32, 3, 3))
#model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
train_size=.75,
random_state=25)
# this is a custom r2 function to use in the model for evaluating performance
def custom_r2(y_true, y_pred):
baseline = K.sum((y_true - K.mean(y_true))**2)
model_fit = K.sum((y_true - y_pred)**2)
return 1. - model_fit / baseline
# Building Neural Net Model:
model = Sequential()
model.add(Dense(1024, input_shape=(21, ), kernel_initializer='normal'))
model.add(Activation('elu'))
model.add(Dropout(.2))
model.add(Dense(1024, kernel_initializer='normal'))
model.add(Activation('elu'))
model.add(Dropout(.5))
model.add(Dense(1, kernel_initializer='normal'))
model.add(Activation('linear'))
model.compile(loss='mean_squared_error', optimizer='adam', metrics=[custom_r2])
def early_stopping_cont_nn(model,
X,
y,
def create_model(input_size, weights=False, summary=True):
# vgg19_model = Sequential()
input_ = Input(shape=input_size)
x = input_
# x =ZeroPadding2D((1, 1),input_shape=(3, 224, 224)))
x = Conv2D(64, (3, 3), padding='same', name='conv1_1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', name='conv1_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
x = Conv2D(128, (3, 3), padding='same', name='conv2_1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', name='conv2_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
x = Conv2D(256, (3, 3), padding='same', name='conv3_1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='conv3_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='conv3_3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='conv3_4')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
x = Conv2D(512, (3, 3), padding='same', name='conv4_1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv4_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv4_3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv4_4')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
x = Conv2D(512, (3, 3), padding='same', name='conv5_1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv5_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv5_3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='conv5_4')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
# Classification layer
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='dense_1')(x)
x = Dropout(0.5)(x)
x = Dense(4096, activation='relu', name='dense_2')(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES, activation='softmax', name='predictions')(x)
# x = Activation("softmax")(x)
return Model(inputs=input_, outputs=x)
if summary:
print(vgg19_model.summary())
def run_model_varyembed(dataset,
numhidden,
hiddendim,
idx2word,
idx2label,
w2v,
basedir,
embedding_dim=400,
validate=True,
num_epochs=30):
train_toks, valid_toks, test_toks, \
train_lex, valid_lex, test_lex, \
train_y, valid_y, test_y = dataset
maxlen = max([len(l) for l in train_lex])
if len(valid_lex) > 0:
maxlen = max(maxlen, max([len(l) for l in valid_lex]))
maxlen = max(maxlen, max([len(l) for l in test_lex]))
vocsize = max(idx2word.keys()) + 1
nclasses = max(idx2label.keys()) + 1
# Pad inputs to max sequence length and turn into one-hot vectors
train_lex = sequence.pad_sequences(train_lex, maxlen=maxlen)
valid_lex = sequence.pad_sequences(valid_lex, maxlen=maxlen)
test_lex = sequence.pad_sequences(test_lex, maxlen=maxlen)
train_y = sequence.pad_sequences(train_y, maxlen=maxlen)
valid_y = sequence.pad_sequences(valid_y, maxlen=maxlen)
test_y = sequence.pad_sequences(test_y, maxlen=maxlen)
train_y = vectorize_set(train_y, maxlen, nclasses)
valid_y = vectorize_set(valid_y, maxlen, nclasses)
test_y = vectorize_set(test_y, maxlen, nclasses)
# Build the model
## BI-DIRECTIONAL
print('Building the model...')
H = numhidden
model = Graph()
model.add_input(name='input', input_shape=[maxlen], dtype='int')
# Add embedding layer
if w2v is None:
model.add_node(Embedding(vocsize,
embedding_dim,
init='lecun_uniform',
input_length=maxlen),
name='embed',
input='input')
else:
embeds = init_embedding_weights(idx2word, w2v)
embed_dim = w2v.wv.syn0.shape[1]
model.add_node(Embedding(vocsize,
embed_dim,
input_length=maxlen,
weights=[embeds],
mask_zero=True),
name='embed',
input='input')
# Build first hidden layer
model.add_node(LSTM(hiddendim, return_sequences=True, activation='tanh'),
name='forward0',
input='embed')
model.add_node(Dropout(0.1), name='dropout0f', input='forward0')
model.add_node(LSTM(hiddendim,
return_sequences=True,
go_backwards=True,
activation='tanh'),
name='backwards0',
input='embed')
model.add_node(Dropout(0.1), name='dropout0b', input='backwards0')
# Build subsequent hidden layers
if H > 1:
for i in range(1, H):
model.add_node(LSTM(hiddendim,
return_sequences=True,
activation='tanh'),
name='forward%d' % i,
input='dropout%df' % (i - 1))
model.add_node(Dropout(0.1),
name='dropout%df' % i,
input='forward%d' % i)
model.add_node(LSTM(hiddendim,
return_sequences=True,
go_backwards=True,
activation='tanh'),
name='backwards%d' % i,
input='dropout%db' % (i - 1))
model.add_node(Dropout(0.1),
name='dropout%db' % i,
input='backwards%d' % i)
# Finish up the network
model.add_node(TimeDistributedDense(nclasses),
name='tdd',
inputs=['dropout%df' % (H - 1),
'dropout%db' % (H - 1)],
merge_mode='ave')
model.add_node(Activation('softmax'), name='softmax', input='tdd')
model.add_output(name='output', input='softmax')
model.compile(optimizer='rmsprop',
loss={'output': 'categorical_crossentropy'})
# Set up callbacks
fileprefix = 'embed_varied_'
am = approximateMatch.ApproximateMatch_SEQ(valid_toks,
valid_y,
valid_lex,
idx2label,
pred_dir=os.path.join(
basedir, 'predictions'),
fileprefix=fileprefix)
mc = callbacks.ModelCheckpoint(
os.path.join(basedir, 'models',
'embedding.model.weights.{epoch:02d}.hdf5'))
cbs = [am, mc]
if validate:
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
patience=3)
cbs.append(early_stopping)
# Train the model
print('Training...')
hist = model.fit({
'input': train_lex,
'output': train_y
},
nb_epoch=num_epochs,
batch_size=1,
validation_data={
'input': valid_lex,
'output': valid_y
},
callbacks=cbs)
if validate:
val_f1, best_model = learning_curve(
hist,
preddir=os.path.join(basedir, 'predictions'),
pltname=os.path.join(
basedir, 'charts',
'hist_varyembed%d_nhidden%d.pdf' % (hiddendim, numhidden)),
fileprefix=fileprefix)
else:
best_model = num_epochs - 1
val_f1 = 0.0
# Save model
json_string = model.to_json()
open(os.path.join(basedir, 'models', 'embedding_model_architecture.json'),
'w').write(json_string)
# Test
bestmodelfile = os.path.join(
basedir, 'models', 'embedding.model.weights.%02d.hdf5' % best_model)
shutil.copyfile(bestmodelfile,
bestmodelfile.replace('.hdf5', '.best.hdf5'))
if validate:
model = model_from_json(
open(
os.path.join(basedir, 'models',
'embedding_model_architecture.json')).read())
model.load_weights(bestmodelfile)
scores = predict_score(model,
test_lex,
test_toks,
test_y,
os.path.join(basedir, 'predictions'),
idx2label,
maxlen,
fileprefix=fileprefix)
scores['val_f1'] = val_f1
return scores, hist.history, best_model
def DenseNet(nb_dense_block=3,
growth_rate=20,
nb_filter=32,
reduction=0.0,
dropout_rate=0.0,
weight_decay=0,
classes=1000,
weights_path=None):
'''Instantiate the DenseNet 121 architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
with tf.device('/gpu:1'):
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(512, 512, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, 224, 224), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 32 ##make this 16
nb_layers = [3, 6, 12, 8] #[6,12,24,16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(nb_filter, (7, 7),
name='conv1',
kernel_initializer='he_normal',
padding='valid')(
x) ### Put strides of 2*2 here and padding = valid
x = BatchNormalization(axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D(
(3, 3), strides=(4, 4),
name='pool1')(x) ### Put strides of 2*2 and pooling 3*3
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
#x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
'''x1 = Conv2D(32,(3,3),dilation_rate = 6, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv1', activation = 'relu',data_format='channels_last')(x)
x2 = Conv2D(32,(3,3),dilation_rate = 12, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv2', activation = 'relu',data_format='channels_last')(x)
x3 = Conv2D(32,(3,3),dilation_rate = 18, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv3', activation = 'relu',data_format='channels_last')(x)
x4 = Conv2D(32,(3,3),dilation_rate = 24, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv4', activation = 'relu',data_format='channels_last')(x)
sum_b = add([x1,x2,x3,x4],name = 'sum_b')'''
#x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x)
x = Conv2D(1, (1, 1), padding='valid',
kernel_initializer='he_normal')(x)
model = Model(img_input, x, name='densenet')
'''xp = model.get_layer(name = 'conv5_blk_scale').output
x1 = Conv2D(16,(3,3),dilation_rate = 6, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv1', activation = 'relu',data_format='channels_last')(xp)
x2 = Conv2D(16,(3,3),dilation_rate = 12, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv2', activation = 'relu',data_format='channels_last')(xp)
x3 = Conv2D(16,(3,3),dilation_rate = 18, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv3', activation = 'relu',data_format='channels_last')(xp)
x4 = Conv2D(16,(3,3),dilation_rate = 24, padding = 'same', kernel_initializer = 'he_normal', name = 'dil_conv4', activation = 'relu',data_format='channels_last')(xp)
sum_b = add([x1,x2,x3,x4],name = 'sum_b')
#sum_b_activation = Activation('relu', name='relu_sum')(sum_b)
out = Conv2D(1,(1,1),padding = 'valid',kernel_initializer='he_normal',activation = 'relu')(sum_b)
model_2 = Model(inputs = model.input, outputs = out) '''
if weights_path is not None:
model.load_weights(weights_path)
return model
def build(width, height, depth, classes):
model = Sequential()
# for TensorFlow <3
inputShape = (height, width, depth)
chanDim = -1
# for other guys
if K.image_data_format() == "channels_first":
inputShape = (depth, height, width)
chanDim = 1
# Conv -> PReLU -> Pool
# 32 filters with 3 x 3 kernel
model.add(Conv2D(32, (3, 3), padding = "same",
input_shape = inputShape))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
# 3 x 3 pool size
model.add(MaxPooling2D(pool_size = (3, 3)))
model.add(Dropout(0.25))
# (Conv -> PReLU) x 2 -> Pool
# MORE FILTERS FOR THE GOD OF FILTERS
model.add(Conv2D(64, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(64, (3, 3), padding = "same"))
# 'twas good, let's repeat
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
# 2 x 2 pool size
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Dropout(0.25))
# (Conv -> PReLU) x 2 -> Pool
# more filters?
model.add(Conv2D(128, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(128, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Dropout(0.25))
# (Conv -> PReLU) x 2 -> Pool
# more filters?
model.add(Conv2D(256, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(256, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(256, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Dropout(0.25))
# (Conv -> PReLU) x 2 -> Pool
# more filters?
model.add(Conv2D(512, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(512, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(Conv2D(512, (3, 3), padding = "same"))
model.add(PReLU())
model.add(BatchNormalization(axis = chanDim))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Dropout(0.25))
# FC -> PReLU -> Softmax
model.add(Flatten())
model.add(Dense(1024))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(1024))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(classes))
model.add(Activation("softmax"))
return model
mask_zero=True,
weights=[embedding_weights]))
# dropout=0.5)) # added dropout
model.add(
LSTM(lstm_output,
input_shape=(
maxlen,
embedding_dim,
),
dropout_W=lstm_dropout_w,
dropout_U=lstm_dropout_u))
#model.add(LSTM(lstm_output, input_shape=(maxlen, embedding_dim,)))
model.add(Dropout(dropout))
model.add(Dense(1))
model.add(
Activation('sigmoid')) # relu is worse, softmax does not work at all
model.summary()
# try using different optimizers and different optimizer configs
print 'Compiling model...'
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# initialize callbacks
history = TrainHistory()
early_stopping = EarlyStopping(monitor='val_acc',
mode='max',
verbose=1,
patience=patience)
batch_size = 64
samples_per_epoch = X_train.shape[0]
### Model
model = Sequential()
model.add(Lambda(lambda x: x / 255. - 0.5,
input_shape=(img_rows, img_cols, 1)))
model.add(
Convolution2D(layer_1_depth,
filter_size_1,
filter_size_1,
border_mode='valid',
subsample=(1, 1)))
model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Convolution2D(layer_2_depth,
filter_size_2,
filter_size_2,
border_mode='valid',
subsample=(1, 1)))
model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(num_neurons_1))
model.add(Activation('elu'))
from keras.models import Sequential
from keras.layers.core import Dense,Activation
from keras.layers.convolutional import Convolution2D
import keras
model = Sequential()
model.add(Dense(4,name='dense1', use_bias=False,input_dim=2))
model.add(Activation('relu',name='relu1'))
model.add(Dense(10, activation='relu',use_bias=0,name='final_layer'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Generate dummy data
import numpy as np
data = np.random.random((1000, 2))
labels = np.random.randint(10, size=(1000, 1))
# Convert labels to categorical one-hot encoding
one_hot_labels = keras.utils.to_categorical(labels, num_classes=10)
# Train the model, iterating on the data in batches of 32 samples
model.fit(data, one_hot_labels, epochs=10, batch_size=32)
print(model.get_layer('dense1').get_weights()[0])
weight_layer1=model.get_layer('dense1').get_weights()[0]
#weight_layer2=model.get_layer('relu1').get_activations()[0]
weight_layer3=model.get_layer('final_layer').get_weights()[0]
weight_layer1_array=[]
weight_layer3_array=[]
def densenet121_model(img_rows,
img_cols,
color_type=1,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-4,
num_classes=None):
'''
DenseNet 121 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfMlRYb3YzV210VzQ
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSTA4SHJVOHNuTXc
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter,
7,
7,
subsample=(2, 2),
name='conv1',
bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = 'imagenet_models/densenet121_weights_th.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = '/home/cnd/.keras/models/densenet121_weights_tf_dim_ordering_tf_kernels.h5'
model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_newfc = Dense(num_classes, name='fc6')(x_newfc)
x_newfc = Activation('softmax', name='prob')(x_newfc)
model = Model(img_input, x_newfc)
# Learning rate is changed to 0.001
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_network(**kwargs):
"""construct a convolutional neural network.
Keword Arguments:
- input_dim: depth of features to be processed by first layer (no default)
- board: width of the go board to be processed (default 19)
- filters_per_layer: number of filters used on every layer (default 128)
- filters_per_layer_K: (where K is between 1 and <layers>) number of filters
used on layer K (default #filters_per_layer)
- layers: number of convolutional steps (default 12)
- filter_width_K: (where K is between 1 and <layers>) width of filter on
layer K (default 3 except 1st layer which defaults to 5).
Must be odd.
"""
defaults = {
"board": 19,
"filters_per_layer": 128,
"layers": 12,
"filter_width_1": 5
}
# copy defaults, but override with anything in kwargs
params = defaults
params.update(kwargs)
# create the network:
# a series of zero-paddings followed by convolutions
# such that the output dimensions are also board x board
network = Sequential()
# create first layer
network.add(
Conv2D(input_shape=(params["input_dim"], params["board"],
params["board"]),
filters=params.get("filters_per_layer_1",
params["filters_per_layer"]),
kernel_size=(params["filter_width_1"],
params["filter_width_1"]),
kernel_initializer='uniform',
activation='relu',
padding='same',
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None))
# create all other layers
for i in range(2, params["layers"] + 1):
# use filter_width_K if it is there, otherwise use 3
filter_key = "filter_width_%d" % i
filter_width = params.get(filter_key, 3)
# use filters_per_layer_K if it is there, otherwise use default value
filter_count_key = "filters_per_layer_%d" % i
filter_nb = params.get(filter_count_key,
params["filters_per_layer"])
network.add(
Conv2D(filters=filter_nb,
kernel_size=(filter_width, filter_width),
kernel_initializer='uniform',
activation='relu',
padding='same',
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None))
# the last layer maps each <filters_per_layer> feature to a number
network.add(
Conv2D(filters=1,
kernel_size=(1, 1),
kernel_initializer='uniform',
padding='same',
kernel_constraint=None,
activity_regularizer=None,
trainable=True,
strides=[1, 1],
use_bias=True,
bias_regularizer=None,
bias_constraint=None,
data_format="channels_first",
kernel_regularizer=None))
# reshape output to be board x board
network.add(Flatten())
# add a bias to each board location
network.add(Bias())
# softmax makes it into a probability distribution
network.add(Activation('softmax'))
return network
Ytrain = Y[:split_point]
Xtest = X[split_point:]
Ytest = Y[split_point:]
# Creating a baseline.
# dummy = DummyClassifier(strategy='stratified')
# dummy.fit(Xtrain, Ytrain)
# Yguess = dummy.predict(Xtest)
# print("### BASELINE")
# print("Accuracy score of baseline: ", accuracy_score(Ytest, Yguess))
# print(classification_report(Ytest, Yguess))
# Define the properties of the perceptron model
model = Sequential()
model.add(Dense(input_dim=X.shape[1], units=Y.shape[1]))
model.add(Activation("tanh"))
sgd = SGD(lr=0.01)
loss_function = 'mean_squared_error'
model.compile(loss=loss_function, optimizer=sgd, metrics=['accuracy'])
# Train the perceptron
# ### CHECKING EPOCHS
# e = 20
# history = model.fit(Xtrain, Ytrain, verbose=1, epochs=e, batch_size=32)
# # Get predictions
# Yguess = model.predict(Xtest)
# # Convert to numerical labels to get scores with sklearn in 6-way setting
# Yguess = numpy.argmax(Yguess, axis = 1)
# the data, shuffled and split between tran and test sets
X_tr_orig = np.load('../other/X_tr_orig.npy')
X_train = X_tr_orig.reshape(X_tr_orig.shape[0], 1, 20, 20)
X_train = X_train.astype("float32")
X_train /= 255
# Model
img_channels = 1
img_rows = img_cols = 20
nb_classes = 62
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='full',
input_shape=(img_channels, img_rows, img_cols)))
convout1 = Activation('relu')
model.add(convout1)
model.add(Convolution2D(32, 3, 3))
convout2 = Activation('relu')
model.add(convout2)
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(64, 3, 3, border_mode='full'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
def build_model(X_train,
row,
cell,
nb_classes=3,
Chans=3,
Samples=75,
dropoutRate=0.25,
kernLength=150,
F1=2,
D=2,
F2=8,
dropoutType='Dropout'):
model = Sequential()
dropoutType = Dropout
model.add(
Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(1, Samples, Chans),
use_bias=False))
model.add(BatchNormalization(axis=1))
model.add(
DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.),
data_format='channels_first'))
model.add(BatchNormalization(axis=1))
model.add(Activation('elu'))
model.add(AveragePooling2D((1, 2), data_format='channels_first'))
model.add(SeparableConv2D(F2, (1, 16), use_bias=False, padding='same'))
model.add(BatchNormalization(axis=1))
model.add(Activation('elu'))
model.add(AveragePooling2D((1, 1)))
model.add(dropoutType(dropoutRate))
model.add(Reshape((2, 8 * 73)))
# model.add(Flatten(name='flatten'))
model.add(LSTM(128, activation='tanh', recurrent_activation='hard_sigmoid', \
use_bias=True, kernel_initializer='glorot_uniform', \
recurrent_initializer='orthogonal', \
unit_forget_bias=True, kernel_regularizer=None, \
recurrent_regularizer=None, \
bias_regularizer=None, activity_regularizer=None, \
kernel_constraint=None, recurrent_constraint=None, \
bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \
implementation=1, return_sequences=True, return_state=False, \
go_backwards=False, stateful=False, unroll=False))
model.add(dropoutType(dropoutRate))
model.add(LSTM(64, activation='tanh', recurrent_activation='hard_sigmoid', \
use_bias=True, kernel_initializer='glorot_uniform', \
recurrent_initializer='orthogonal', \
unit_forget_bias=True, kernel_regularizer=None, \
recurrent_regularizer=None, \
bias_regularizer=None, activity_regularizer=None, \
kernel_constraint=None, recurrent_constraint=None, \
bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \
implementation=1, return_sequences=False, return_state=False, \
go_backwards=False, stateful=False, unroll=False))
model.add(Dense(nb_classes, name='dense',
kernel_constraint=max_norm(0.25)))
model.add(Activation('softmax', name='softmax'))
opt = optimizers.adam(lr=0.001)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=['accuracy'])
plot_model(model, to_file='model.png', show_shapes=True)
model.summary()
return model
