# CNN model
tensorboard = TensorBoard(log_dir=f'logs/{NAME}')
model = Sequential()
model.add(Conv2D(30, (5, 5), input_shape=X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(50, (5, 5)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(500))
model.add(Activation("relu"))
model.add(Dense(100))
model.add(Activation("tanh"))
model.add(Dense(27))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=['accuracy'])
model.fit(X,
y,
batch_size=32,
dataset = numpy.loadtxt("prima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables, splitting csv data
X = dataset[:, 0:8]
Y = dataset[:, 8]
# split X, Y into a train and test set
x_train, x_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.2,
random_state=42)
# create model, add dense layers one by one specifying activation function
model = Sequential()
# input layer requires input_dim param
model.add(Dense(15, input_dim=8, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dropout(.2))
# sigmoid instead of relu for final probability between 0 and 1
model.add(Dense(1, activation='sigmoid'))
# compile the model, adam gradient descent (optimized)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=['accuracy'])
# call the function to fit to the data (training the network)
model.fit(x_train,
y_train,
epochs=1000,
plt.plot(seq)
plt.show()
validation_data = (np.expand_dims(x_test_scaled,
axis=0), np.expand_dims(y_test_scaled,
axis=0))
#Create the Recurrent Neural Network
model = Sequential()
model.add(
GRU(units=512, return_sequences=True, input_shape=(
None,
num_x_signals,
)))
model.add(Dense(num_y_signals, activation='sigmoid'))
if False:
# Maybe use lower init-ranges.
init = RandomUniform(minval=-0.05, maxval=0.05)
model.add(
Dense(num_y_signals, activation='linear', kernel_initializer=init))
warmup_steps = 20
def loss_mse_warmup(y_true, y_pred):
"""
Calculate the Mean Squared Error between y_true and y_pred,
but ignore the beginning "warmup" part of the sequences.
def DCN(
feature_dim_dict,
embedding_size='auto',
cross_num=2,
hidden_size=[
128,
128,
],
l2_reg_embedding=1e-5,
l2_reg_cross=1e-5,
l2_reg_deep=0,
init_std=0.0001,
seed=1024,
keep_prob=1,
use_bn=False,
activation='relu',
final_activation='sigmoid',
):
"""Instantiates the Deep&Cross Network architecture.
:param feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param embedding_size: positive int or str,sparse feature embedding_size.If set to "auto",it will be 6*pow(cardinality,025)
:param cross_num: positive integet,cross layer number
:param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param l2_reg_cross: float. L2 regularizer strength applied to cross net
:param l2_reg_deep: float. L2 regularizer strength applied to deep net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param keep_prob: float in (0,1]. keep_prob used in deep net
:param use_bn: bool. Whether use BatchNormalization before activation or not.in deep net
:param activation: Activation function to use in deep net
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:return: A Keras model instance.
"""
if len(hidden_size) == 0 and cross_num == 0:
raise ValueError("Either hidden_layer or cross layer must > 0")
if not isinstance(
feature_dim_dict, dict
) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}"
)
sparse_input, dense_input = get_input(
feature_dim_dict,
None,
)
sparse_embedding = get_embeddings(feature_dim_dict, embedding_size,
init_std, seed, l2_reg_embedding)
embed_list = [
sparse_embedding[i](sparse_input[i]) for i in range(len(sparse_input))
]
deep_input = Flatten()(Concatenate()(embed_list))
if len(dense_input) > 0:
if len(dense_input) == 1:
continuous_list = dense_input[0]
else:
continuous_list = Concatenate()(dense_input)
deep_input = Concatenate()([deep_input, continuous_list])
if len(hidden_size) > 0 and cross_num > 0: # Deep & Cross
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input)
cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(deep_input)
stack_out = Concatenate()([cross_out, deep_out])
final_logit = Dense(1, use_bias=False, activation=None)(stack_out)
elif len(hidden_size) > 0: # Only Deep
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input)
final_logit = Dense(1, use_bias=False, activation=None)(deep_out)
elif cross_num > 0: # Only Cross
cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(deep_input)
final_logit = Dense(1, use_bias=False, activation=None)(cross_out)
else: # Error
raise NotImplementedError
# Activation(self.final_activation)(final_logit)
output = PredictionLayer(final_activation)(final_logit)
model = Model(inputs=sparse_input + dense_input, outputs=output)
return model
X_train = np.array(X_train)
X_test = np.array(X_test)
# One-Hot encoding for classes
y_train_saved = y_train
y_train = np.array(keras.utils.to_categorical(y_train, 2))
y_test_saved = y_test
y_test = np.array(keras.utils.to_categorical(y_test, 2))
print(X_train.shape)
print(X_test.shape)
#keras.normalise can be used
model = Sequential()
model.add(TimeDistributed(Dense(128), input_shape=(100, 320)))
model.add(Activation('relu'))
model.add(Dropout(rate=0.5))
model.add(TimeDistributed(Dense(128)))
model.add(Activation('relu'))
model.add(Dropout(rate=0.5))
model.add(TimeDistributed(Dense(128), name='dense_inter'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.5))
model.add(TimeDistributed(Dense(2)))
model.add(Activation('sigmoid'))
time_distributed_merge_layer = tf.keras.layers.Lambda(
def train_CNN(combined_train, Y_train, combined_test, Y_test, batch_size,
drop_out, lr, epochs):
'''
Input X_train,X_test: shape (number of sequences,4,2,41)
Y_train,Y_test: (number of sequences,)
batch_size: batch size for training LSTM
drop_out: drop_out rate for LSTM layers
lr: learning rate of Adam optimizer
epochs: training epochs
This function define the structure of CNN-LSTM model and trains with Adam optimizer.
Output: a real number of average of validation MSE of 3-fold cross validation.
'''
def create_CNN(serie_size, n_features, drop_out):
inputs = Input(shape=(serie_size, n_features))
x = (Conv1D(64,
kernel_size=2,
input_shape=(serie_size, n_features),
padding='same',
activation='relu'))(inputs)
x = (Dropout(drop_out))(x)
x = (Conv1D(32, kernel_size=2, padding='same', activation='relu'))(x)
x = (MaxPooling1D(pool_size=2))(x)
x = (Dropout(drop_out))(x)
model = Model(inputs, x)
return model
kf = KFold(n_splits=3)
score = []
for train_index, test_index in kf.split(combined_train[0]):
x_train, x_valid = [i[train_index] for i in combined_train
], [i[test_index] for i in combined_train]
print(len(x_train))
print(len(x_train[0][0]))
y_train, y_valid = np.array(Y_train)[train_index], np.array(
Y_train)[test_index]
temp_1 = create_CNN(2, 41, drop_out)
temp_3 = create_CNN(2, 41, drop_out)
temp_2 = create_CNN(2, 41, drop_out)
temp_4 = create_CNN(2, 41, drop_out)
optimizer = optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
decay=0.01)
init = glorot_normal(seed=None)
init1 = RandomUniform(minval=-0.05, maxval=0.05)
## 10 * 32
combinedInput = concatenate(
[temp_1.output, temp_2.output, temp_3.output, temp_4.output])
combinedInput = Reshape((4, 32), input_shape=(4 * 32, ))(combinedInput)
lstm = LSTM(units=16,
dropout=drop_out,
recurrent_dropout=drop_out,
return_sequences=False,
kernel_initializer=init)
output = lstm(combinedInput)
output = Dense(1, activation='linear',
kernel_initializer=init1)(output)
model = Model(
inputs=[temp_1.input, temp_2.input, temp_3.input, temp_4.input],
outputs=output)
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.summary()
train_history = model.fit(x=[i for i in x_train],
y=y_train,
epochs=epochs,
validation_data=(x_valid, y_valid),
batch_size=batch_size)
val_loss = train_history.history['val_loss'][-1]
score.append(val_loss)
return (np.mean(score))
def EEGNet_old(nb_classes,
Chans=64,
Samples=128,
regRate=0.0001,
dropoutRate=0.25,
kernels=[(2, 32), (8, 4)],
strides=(2, 4)):
""" Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2)
This model is the original EEGNet model proposed on arxiv
https://arxiv.org/abs/1611.08024v2
with a few modifications: we use striding instead of max-pooling as this
helped slightly in classification performance while also providing a
computational speed-up.
Note that we no longer recommend the use of this architecture, as the new
version of EEGNet performs much better overall and has nicer properties.
Inputs:
nb_classes : total number of final categories
Chans, Samples : number of EEG channels and samples, respectively
regRate : regularization rate for L1 and L2 regularizations
dropoutRate : dropout fraction
kernels : the 2nd and 3rd layer kernel dimensions (default is
the [2, 32] x [8, 4] configuration)
strides : the stride size (note that this replaces the max-pool
used in the original paper)
"""
# start the model
input_main = Input((1, Chans, Samples))
layer1 = Conv2D(16, (Chans, 1),
input_shape=(1, Chans, Samples),
kernel_regularizer=l1_l2(l1=regRate,
l2=regRate))(input_main)
layer1 = BatchNormalization(axis=1)(layer1)
layer1 = Activation('elu')(layer1)
layer1 = Dropout(dropoutRate)(layer1)
permute_dims = 2, 1, 3
permute1 = Permute(permute_dims)(layer1)
layer2 = Conv2D(4,
kernels[0],
padding='same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides=strides)(permute1)
layer2 = BatchNormalization(axis=1)(layer2)
layer2 = Activation('elu')(layer2)
layer2 = Dropout(dropoutRate)(layer2)
layer3 = Conv2D(4,
kernels[1],
padding='same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides=strides)(layer2)
layer3 = BatchNormalization(axis=1)(layer3)
layer3 = Activation('elu')(layer3)
layer3 = Dropout(dropoutRate)(layer3)
flatten = Flatten(name='flatten')(layer3)
dense = Dense(nb_classes, name='dense')(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def DIN(
feature_dim_dict,
seq_feature_list,
embedding_size=8,
hist_len_max=16,
use_din=True,
use_bn=False,
hidden_size=(200, 80),
activation='relu',
att_hidden_size=(80, 40),
att_activation=Dice,
att_weight_normalization=False,
l2_reg_deep=0,
l2_reg_embedding=1e-5,
final_activation='sigmoid',
keep_prob=1,
init_std=0.0001,
seed=1024,
):
"""Instantiates the Deep Interest Network architecture.
:param feature_dim_dict: dict,to indicate sparse field (**now only support sparse feature**)like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':[]}
:param seq_feature_list: list,to indicate sequence sparse field (**now only support sparse feature**),must be a subset of ``feature_dim_dict["sparse"]``
:param embedding_size: positive integer,sparse feature embedding_size.
:param hist_len_max: positive int, to indicate the max length of seq input
:param use_din: bool, whether use din pooling or not.If set to ``False``,use **sum pooling**
:param use_bn: bool. Whether use BatchNormalization before activation or not in deep net
:param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param activation: Activation function to use in deep net
:param att_hidden_size: list,list of positive integer , the layer number and units in each layer of attention net
:param att_activation: Activation function to use in attention net
:param att_weight_normalization: bool.Whether normalize the attention score of local activation unit.
:param l2_reg_deep: float. L2 regularizer strength applied to deep net
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:param keep_prob: float in (0,1]. keep_prob used in deep net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:return: A Keras model instance.
"""
for feature_dim_dict in [feature_dim_dict]:
if not isinstance(
feature_dim_dict, dict
) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}"
)
if len(feature_dim_dict['dense']) > 0:
raise ValueError('Now DIN only support sparse input')
sparse_input, user_behavior_input, user_behavior_length = get_input(
feature_dim_dict, seq_feature_list, hist_len_max)
sparse_embedding_dict = {
feat: Embedding(feature_dim_dict["sparse"][feat],
embedding_size,
embeddings_initializer=RandomNormal(mean=0.0,
stddev=init_std,
seed=seed),
embeddings_regularizer=l2(l2_reg_embedding),
name='sparse_emb_' + str(i) + '-' + feat)
for i, feat in enumerate(feature_dim_dict["sparse"])
}
query_emb_list = [
sparse_embedding_dict[feat](sparse_input[feat])
for feat in seq_feature_list
]
keys_emb_list = [
sparse_embedding_dict[feat](user_behavior_input[feat])
for feat in seq_feature_list
]
deep_input_emb_list = [
sparse_embedding_dict[feat](sparse_input[feat])
for feat in feature_dim_dict["sparse"]
]
query_emb = Concatenate()(
query_emb_list) if len(query_emb_list) > 1 else query_emb_list[0]
keys_emb = Concatenate()(
keys_emb_list) if len(keys_emb_list) > 1 else keys_emb_list[0]
deep_input_emb = Concatenate()(deep_input_emb_list) if len(
deep_input_emb_list) > 1 else deep_input_emb_list[0]
if use_din:
hist = AttentionSequencePoolingLayer(
att_hidden_size,
att_activation,
weight_normalization=att_weight_normalization)(
[query_emb, keys_emb, user_behavior_length])
else:
hist = SequencePoolingLayer(hist_len_max,
'sum')([keys_emb, user_behavior_length])
deep_input_emb = Concatenate()([deep_input_emb, hist])
output = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input_emb)
output = Dense(1, final_activation)(output)
output = Reshape([1])(output)
model_input_list = list(sparse_input.values()) + list(
user_behavior_input.values()) + [user_behavior_length]
model = Model(inputs=model_input_list, outputs=output)
return model
def VGG_16(spatial_size,
classes,
channels,
channel_first=True,
weights_path=None):
model = Sequential()
if channel_first:
model.add(
ZeroPadding2D((1, 1),
input_shape=(channels, spatial_size, spatial_size)))
else:
model.add(
ZeroPadding2D((1, 1),
input_shape=(spatial_size, spatial_size, channels)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2))) # 33
model.add(Flatten())
model.add(Dense(4096, activation='relu')) # 34
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu')) # 35
model.add(Dropout(0.5))
model.add(Dense(2622, activation='softmax')) # Dropped
if weights_path:
model.load_weights(weights_path)
model.pop()
model.add(Dense(classes, activation='softmax')) # 36
return model
def create_VGG_net(self, raw=120, column=320, channel=1):
print('create VGG model!!')
inputShape = (raw, column, channel)
init = 'he_normal'
# init = 'glorot_normal'
activation = 'relu'
keep_prob_conv = 0.25
keep_prob_dense = 0.5
chanDim = -1
classes = 8
model = Sequential()
# CONV => RELU => POOL
model.add(Conv2D(32, (3, 3), padding="same", input_shape=inputShape, kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(3, 3)))
model.add(Dropout(keep_prob_conv))
# (CONV => RELU) * 2 => POOL
model.add(Conv2D(64, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(Conv2D(64, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(keep_prob_conv))
# (CONV => RELU) * 2 => POOL
model.add(Conv2D(128, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(Conv2D(128, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(keep_prob_conv))
# (CONV => RELU) * 2 => POOL
model.add(Conv2D(128, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(Conv2D(128, (3, 3), padding="same", kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(keep_prob_conv))
# first (and only) set of FC => RELU layers
model.add(Flatten())
model.add(Dense(1024, kernel_initializer=init))
model.add(Activation(activation))
model.add(BatchNormalization())
model.add(Dropout(keep_prob_dense))
# softmax classifier
model.add(Dense(classes))
model.add(Activation("softmax"))
# return the constructed network architecture
self.model = model
help="dimension of model ouput")
args = parser.parse_args()
dataloader = Dataloader(args)
p = PointEmbedding(args)
p_model = p.build()
loss_func = p.custom_loss()
x1 = Input((28, 28))
x2 = Input((28, 28))
x1_r = K.reshape(x1, (-1, 28, 28, 1))
x2_r = K.reshape(x2, (-1, 28, 28, 1))
pair1 = K.concatenate([x1_r, x2_r], axis=2)
x3 = Input((28, 28))
x4 = Input((28, 28))
x3_r = K.reshape(x3, (-1, 28, 28, 1))
x4_r = K.reshape(x4, (-1, 28, 28, 1))
pair2 = K.concatenate([x3_r, x4_r], axis=2)
output = Conv2D(1, (5, 5), (2, 2))(pair1)
output = MaxPooling2D((2, 2), (1, 1))(output)
output = Conv2D(1, (5, 5), (2, 2))(output)
output = MaxPooling2D((2, 2), (1, 1))(output)
output = Flatten()(output)
output = Dense(2)(output)
print(K.square(output))
print(K.sum(K.square(output), axis=-1))
print(K.sigmoid(K.sum(K.square(output), axis=-1)))
yp = dfprod[:,1:] xc = dfcont[:,0:-1] yc = dfcont[:,1:] z = 1000000 xptrain = xp[0:z,:] yptrain = yp[0:z,:] xptest = xp[z:,:] yptest = yp[z:,:] vocab = np.max(dfprod)+1 inp = Input(shape=(19,)) emb = Embedding(input_dim=vocab,mask_zero=True,output_dim= 100, input_length=len(xp[0])) (inp) g = GRU(units=100, return_sequences= True) (emb) d = Dense(vocab, activation='softmax')(g) model = Model(inputs=inp, outputs=d) optimizeradam = tf.train.AdamOptimizer() model.compile(loss='categorical_crossentropy', optimizer=optimizeradam, metrics= [tkca]) categorical_labels = to_categorical(yptrain) categorical_labels2 = to_categorical(yptest) model.fit(xptrain,categorical_labels, epochs= 5, batch_size= 1) print(model.evaluate(xptest,categorical_labels2)) # model = Sequential() # model.add(Embedding(input_dim=vocab,mask_zero=True,output_dim= 100, input_length=len(xp[0]))) # model.add(GRU(units=100, return_sequences= True)) # model.add(Dense(vocab, activation='softmax')) # categorical_labels = to_categorical(yp) # model.compile(loss='categorical_crossentropy', optimizer='adam', metrics= ['acc']) # model.summary()
def EEGNet(nb_classes,
Chans=64,
Samples=128,
dropoutRate=0.5,
kernLength=64,
F1=8,
D=2,
F2=16,
norm_rate=0.25,
dropoutType='Dropout'):
""" Keras Implementation of EEGNet
http://iopscience.iop.org/article/10.1088/1741-2552/aace8c/meta
Note that this implements the newest version of EEGNet and NOT the earlier
version (version v1 and v2 on arxiv). We strongly recommend using this
architecture as it performs much better and has nicer properties than
our earlier version. For example:
1. Depthwise Convolutions to learn spatial filters within a
temporal convolution. The use of the depth_multiplier option maps
exactly to the number of spatial filters learned within a temporal
filter. This matches the setup of algorithms like FBCSP which learn
spatial filters within each filter in a filter-bank. This also limits
the number of free parameters to fit when compared to a fully-connected
convolution.
2. Separable Convolutions to learn how to optimally combine spatial
filters across temporal bands. Separable Convolutions are Depthwise
Convolutions followed by (1x1) Pointwise Convolutions.
While the original paper used Dropout, we found that SpatialDropout2D
sometimes produced slightly better results for classification of ERP
signals. However, SpatialDropout2D significantly reduced performance
on the Oscillatory dataset (SMR, BCI-IV Dataset 2A). We recommend using
the default Dropout in most cases.
Assumes the input signal is sampled at 128Hz. If you want to use this model
for any other sampling rate you will need to modify the lengths of temporal
kernels and average pooling size in blocks 1 and 2 as needed (double the
kernel lengths for double the sampling rate, etc). Note that we haven't
tested the model performance with this rule so this may not work well.
The model with default parameters gives the EEGNet-8,2 model as discussed
in the paper. This model should do pretty well in general, although it is
advised to do some model searching to get optimal performance on your
particular dataset.
We set F2 = F1 * D (number of input filters = number of output filters) for
the SeparableConv2D layer. We haven't extensively tested other values of this
parameter (say, F2 < F1 * D for compressed learning, and F2 > F1 * D for
overcomplete). We believe the main parameters to focus on are F1 and D.
Inputs:
nb_classes : int, number of classes to classify
Chans, Samples : number of channels and time points in the EEG data
dropoutRate : dropout fraction
kernLength : length of temporal convolution in first layer. We found
that setting this to be half the sampling rate worked
well in practice. For the SMR dataset in particular
since the data was high-passed at 4Hz we used a kernel
length of 32.
F1, F2 : number of temporal filters (F1) and number of pointwise
filters (F2) to learn. Default: F1 = 8, F2 = F1 * D.
D : number of spatial filters to learn within each temporal
convolution. Default: D = 2
dropoutType : Either SpatialDropout2D or Dropout, passed as a string.
"""
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape=(1, Chans, Samples))
##################################################################
block1 = Conv2D(F1, (1, kernLength), padding='same',
use_bias=False)(input1)
block1 = BatchNormalization(axis=1)(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.))(block1)
block1 = BatchNormalization(axis=1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 4))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16), padding='same',
use_bias=False)(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 8))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(nb_classes,
name='dense',
kernel_constraint=max_norm(norm_rate))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def DeepConvNet(nb_classes, Chans=64, Samples=256, dropoutRate=0.5):
""" Keras implementation of the Deep Convolutional Network as described in
Schirrmeister et. al. (2017), Human Brain Mapping.
This implementation assumes the input is a 2-second EEG signal sampled at
128Hz, as opposed to signals sampled at 250Hz as described in the original
paper. We also perform temporal convolutions of length (1, 5) as opposed
to (1, 10) due to this sampling rate difference.
Note that we use the max_norm constraint on all convolutional layers, as
well as the classification layer. We also change the defaults for the
BatchNormalization layer. We used this based on a personal communication
with the original authors.
ours original paper
pool_size 1, 2 1, 3
strides 1, 2 1, 3
conv filters 1, 5 1, 10
Note that this implementation has not been verified by the original
authors.
"""
# start the model
input_main = Input((1, Chans, Samples))
block1 = Conv2D(25, (1, 5),
input_shape=(1, Chans, Samples),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(input_main)
block1 = Conv2D(25, (Chans, 1),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block1)
block1 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block1)
block1 = Activation('elu')(block1)
block1 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block1)
block1 = Dropout(dropoutRate)(block1)
block2 = Conv2D(50, (1, 5),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block1)
block2 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block2)
block2 = Activation('elu')(block2)
block2 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block2)
block2 = Dropout(dropoutRate)(block2)
block3 = Conv2D(100, (1, 5),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block2)
block3 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block3)
block3 = Activation('elu')(block3)
block3 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block3)
block3 = Dropout(dropoutRate)(block3)
block4 = Conv2D(200, (1, 5),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block3)
block4 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block4)
block4 = Activation('elu')(block4)
block4 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block4)
block4 = Dropout(dropoutRate)(block4)
flatten = Flatten()(block4)
dense = Dense(nb_classes, kernel_constraint=max_norm(0.5))(flatten)
softmax = Activation('softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def train_Combined(combined_train, Y_train, combined_test, Y_test, batch_size,
drop_out, lr, epochs):
'''
Input X_train,X_test: shape (10,number of sequences,17/24)
Y_train,Y_test: (number of sequences,)
batch_size: batch size for training LSTM
drop_out: drop_out rate for LSTM layers
lr: learning rate of Adam optimizer
epochs: training epochs
This function define the structure of MultiInput-LSTM model and trains with Adam optimizer.
Output: a real number of average of validation MSE of 3-fold cross validation.
'''
def create_dense(serie_size, n_features, drop_out):
inputs = Input(shape=(serie_size, n_features))
x = Dense(10, activation='relu')(inputs)
x = Dense(8, activation='relu')(x)
x = Flatten()(x)
model = Model(inputs, x)
return model
kf = KFold(n_splits=3)
score = []
for train_index, test_index in kf.split(combined_train[0]):
print(train_index)
x_train, x_valid = [i[train_index] for i in combined_train
], [i[test_index] for i in combined_train]
y_train, y_valid = np.array(Y_train)[train_index], np.array(
Y_train)[test_index]
serie_size = 1
n_features_gate = len(combined_train[0][0][0])
n_features_net = len(combined_train[1][0][0])
print(n_features_net)
gate_1 = create_dense(serie_size, n_features_gate, drop_out)
gate_3 = create_dense(serie_size, n_features_gate, drop_out)
gate_5 = create_dense(serie_size, n_features_gate, drop_out)
gate_7 = create_dense(serie_size, n_features_gate, drop_out)
gate_9 = create_dense(serie_size, n_features_gate, drop_out)
net_2 = create_dense(serie_size, n_features_net, drop_out)
net_4 = create_dense(serie_size, n_features_net, drop_out)
net_6 = create_dense(serie_size, n_features_net, drop_out)
net_8 = create_dense(serie_size, n_features_net, drop_out)
net_10 = create_dense(serie_size, n_features_net, drop_out)
optimizer = optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
decay=0.01)
init = glorot_normal(seed=None)
init1 = RandomUniform(minval=-0.05, maxval=0.05)
## 10 * 32
print(gate_1.output)
combinedInput = concatenate([gate_1.output, net_2.output,gate_3.output,net_4.output,gate_5.output,net_6.output,\
gate_7.output,net_8.output,gate_9.output,net_10.output])
combinedInput = Reshape((10, 8), input_shape=(8 * 10, ))(combinedInput)
lstm = LSTM(units=16,
dropout=drop_out,
recurrent_dropout=drop_out,
return_sequences=False,
kernel_initializer=init)
output = lstm(combinedInput)
output = Dense(1, activation='linear',
kernel_initializer=init1)(output)
model = Model(inputs=[gate_1.input, net_2.input,gate_3.input,net_4.input,gate_5.input,net_6.input\
,gate_7.input,net_8.input,gate_9.input,net_10.input], outputs=output)
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.summary()
train_history = model.fit(x=[i for i in x_train],
y=y_train,
epochs=epochs,
validation_data=(x_valid, y_valid),
batch_size=batch_size)
val_loss = train_history.history['val_loss'][-1]
score.append(val_loss)
return (np.mean(score))
def AFM(
feature_dim_dict,
embedding_size=8,
use_attention=True,
attention_factor=8,
l2_reg_linear=1e-5,
l2_reg_embedding=1e-5,
l2_reg_att=1e-5,
keep_prob=1.0,
init_std=0.0001,
seed=1024,
final_activation='sigmoid',
):
"""Instantiates the Attentonal Factorization Machine architecture.
:param feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param embedding_size: positive integer,sparse feature embedding_size
:param use_attention: bool,whether use attention or not,if set to ``False``.it is the same as **standard Factorization Machine**
:param attention_factor: positive integer,units in attention net
:param l2_reg_linear: float. L2 regularizer strength applied to linear part
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param l2_reg_att: float. L2 regularizer strength applied to attention net
:param keep_prob: float in (0,1]. keep_prob after attention net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:return: A Keras model instance.
"""
if not isinstance(
feature_dim_dict, dict
) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim_dict must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}"
)
if not isinstance(feature_dim_dict["sparse"], dict):
raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is",
type(feature_dim_dict['sparse']))
if not isinstance(feature_dim_dict["dense"], list):
raise ValueError("feature_dim_dict['dense'] must be a list,cur is",
type(feature_dim_dict['dense']))
sparse_input, dense_input = get_input(feature_dim_dict, None)
sparse_embedding, linear_embedding, = get_embeddings(
feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding,
l2_reg_linear)
embed_list = [
sparse_embedding[i](sparse_input[i]) for i in range(len(sparse_input))
]
linear_term = [
linear_embedding[i](sparse_input[i]) for i in range(len(sparse_input))
]
if len(linear_term) > 1:
linear_term = add(linear_term)
elif len(linear_term) > 0:
linear_term = linear_term[0]
else:
linear_term = 0
if len(dense_input) > 0:
continuous_embedding_list = list(
map(
Dense(
embedding_size,
use_bias=False,
kernel_regularizer=l2(l2_reg_embedding),
), dense_input))
continuous_embedding_list = list(
map(Reshape((1, embedding_size)), continuous_embedding_list))
embed_list += continuous_embedding_list
dense_input_ = dense_input[0] if len(
dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(
1,
activation=None,
use_bias=False,
kernel_regularizer=l2(l2_reg_linear))(dense_input_)
linear_term = add([linear_dense_logit, linear_term])
fm_input = Concatenate(axis=1)(embed_list)
if use_attention:
fm_out = AFMLayer(attention_factor, l2_reg_att, keep_prob,
seed)(embed_list)
else:
fm_out = FM()(fm_input)
final_logit = add([linear_term, fm_out])
output = PredictionLayer(final_activation)(final_logit)
model = Model(inputs=sparse_input + dense_input, outputs=output)
return model
def train_single_LSTM(X_train, Y_train, X_test, Y_test, batch_size, drop_out,
lr, epochs):
'''
Input X_train,X_test: shape(number of sequences,4 sliding window,(24+17)*2 features)
Y_train,Y_test: (number of sequences,1)
batch_size: batch size for training LSTM
drop_out: drop_out rate for LSTM layers
lr: learning rate of Adam optimizer
epochs: training epochs
This function define the structure of LSTM model and trains with Adam optimizer.
Output: a real number of average of validation MSE of 3-fold cross validation.
'''
serie_size = len(X_train[0]) # 4
n_features = len(X_train[0][0]) # 82
# epochs = parameter_dict['epochs']
# batch = parameter_dict['batch']
# lr = parameter_dict['lr']
## input shape : samples, time steps, and features.
lstm_model = Sequential()
optimizer = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, decay=0.01)
#optimizer = RMSprop(lr=0.0005, rho=0.9, epsilon=None, decay=0.0)
init = glorot_normal(seed=None)
init1 = RandomUniform(minval=-0.05, maxval=0.05)
lstm_model.add(
LSTM(units=128,
dropout=drop_out,
recurrent_dropout=drop_out,
input_shape=(serie_size, n_features),
return_sequences=True,
kernel_initializer=init))
lstm_model.add(
LSTM(units=64,
dropout=drop_out,
recurrent_dropout=drop_out,
return_sequences=False,
kernel_initializer=init))
lstm_model.add(Dense(1, activation='linear', kernel_initializer=init1))
lstm_model.compile(loss='mean_squared_error', optimizer=optimizer)
lstm_model.summary()
kf = KFold(n_splits=3)
score = []
for train_index, test_index in kf.split(X_train):
x_train, x_valid = np.array(X_train)[train_index], np.array(
X_train)[test_index]
y_train, y_valid = np.array(Y_train)[train_index], np.array(
Y_train)[test_index]
x_train = np.asarray(x_train)
y_train = np.asarray(y_train)
x_valid = np.asarray(x_valid)
y_valid = np.asarray(y_valid)
x_test = np.asarray(X_test)
train_history = lstm_model.fit(x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_valid, y_valid),
verbose=0,
shuffle=True)
loss = train_history.history['loss']
val_loss = train_history.history['val_loss']
plt.plot(loss)
plt.plot(val_loss)
plt.legend(['loss', 'val_loss'])
plt.show()
# lstm_model.reset_states()
lstm_model.save('my_model.h5')
model = load_model('my_model.h5')
x_test = np.asarray(X_test)
#predicted = model.predict(x_test,batch_size=1)
score.append(val_loss[-1])
return np.mean(score)
test_size=0.2,
random_state=0)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=0.2,
random_state=0)
import tensorflow
import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense
classifier = Sequential()
classifier.add(
Dense(83, kernel_initializer="uniform", activation='relu', input_dim=166))
classifier.add(Dense(83, kernel_initializer="uniform", activation='relu'))
classifier.add(Dense(83, kernel_initializer="uniform", activation='relu'))
classifier.add(
Dense(1, kernel_initializer="uniform", activation='sigmoid',
input_dim=166))
classifier.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
r = classifier.fit(x_train,
y_train,
batch_size=10,
epochs=80,
validation_data=(x_val, y_val))
print(r.history.keys())
model = Sequential()
model.add(Convolution2D(8, 3, 1, activation='relu', input_shape=(100,100,1),padding = 'same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Convolution2D(16, 3, 1, activation='relu', padding = 'same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Convolution2D(32, 3, 1, activation='relu', padding = 'same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(3, activation='softmax'))
model.compile(loss='mean_squared_error',
optimizer = 'Adam',
metrics=['accuracy'])
# 9. Fit model on training data
history = model.fit(X_train, Y_train,
batch_size=32, epochs=100, verbose=1, validation_data = (X_test, Y_test))
# 10. Evaluate model on test data
score = model.evaluate(X_test, Y_test, verbose=0)
# 11. Our scoring
print('This network scored an average val_acc of ' + str(np.mean(history.history['val_acc'][-10:])) + ' in the last 10 epochs')
def EEGNet_SSVEP(nb_classes=12,
Chans=8,
Samples=256,
dropoutRate=0.5,
kernLength=256,
F1=96,
D=1,
F2=96,
dropoutType='Dropout'):
""" SSVEP Variant of EEGNet, as used in [1].
Inputs:
nb_classes : int, number of classes to classify
Chans, Samples : number of channels and time points in the EEG data
dropoutRate : dropout fraction
kernLength : length of temporal convolution in first layer
F1, F2 : number of temporal filters (F1) and number of pointwise
filters (F2) to learn.
D : number of spatial filters to learn within each temporal
convolution.
dropoutType : Either SpatialDropout2D or Dropout, passed as a string.
[1]. Waytowich, N. et. al. (2018). Compact Convolutional Neural Networks
for Classification of Asynchronous Steady-State Visual Evoked Potentials.
Journal of Neural Engineering vol. 15(6).
http://iopscience.iop.org/article/10.1088/1741-2552/aae5d8
"""
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape=(1, Chans, Samples))
##################################################################
block1 = Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(1, Chans, Samples),
use_bias=False)(input1)
block1 = BatchNormalization(axis=1)(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.))(block1)
block1 = BatchNormalization(axis=1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 4))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16), use_bias=False,
padding='same')(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 8))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(nb_classes, name='dense')(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
