def single_nn(opt, dropout, a_func, reg_all, preds_reg, embed_dim, dense_units):
user_embeddings = Embedding(u_cnt,
embed_dim,
embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1),
embeddings_regularizer=l2(reg_all),
input_length=1,
trainable=True, name='user_embeddings')
song_embeddings = Embedding(s_cnt,
embed_dim,
embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1),
embeddings_regularizer=l2(reg_all),
input_length=1,
trainable=True,name='item_embeddings')
uid_input = Input(shape=(1,), dtype='int32')
embedded_usr = user_embeddings(uid_input)
embedded_usr = Reshape((embed_dim,),name='user_embeddings_reshaped')(embedded_usr)
ub = create_bias(uid_input, u_cnt, reg_all,'user_biases')
sid_input = Input(shape=(1,), dtype='int32')
embedded_song = song_embeddings(sid_input)
embedded_song = Reshape((embed_dim,),name='item_embeddings_reshaped')(embedded_song)
mb = create_bias(sid_input, s_cnt, reg_all,'item_biases')
preds = dot([embedded_usr, embedded_song], axes=1)
preds = concatenate([embedded_usr, embedded_song, preds], name='concatenated_embeddings_all')
if (a_func == 'relu') or (a_func == 'elu'):
preds = Dense(1, use_bias=True, activation=a_func, kernel_initializer=RandomNormal(mean=0.0, stddev=0.1), kernel_regularizer=regularizers.l2(preds_reg), bias_initializer=RandomNormal(mean=0.0, stddev=0.1),bias_regularizer=regularizers.l2(preds_reg), name='main_output')(preds)
preds=Dropout(dropout)(preds)
elif a_func == 'selu':
preds = Dense(1, activation=a_func, kernel_initializer=lecun_normal(), name='main_output')(preds)
preds=AlphaDropout(dropout)(preds)
preds = add([ub, preds])
preds = add([mb, preds])
model = Model(inputs=[uid_input, sid_input], outputs=preds)
opt = eval(opt)
model.compile(loss='mse', optimizer=opt, metrics=[mae])
return model
def getModel(outputs=2, sz=32, dropout=0.05, full=True):
inputdata = Input((1,32,32,32))
c1 = conv3d(sz,kernel_size=3)(inputdata)
sred1 = spred(c1,(16,4,5,7))
cres1 = resiconv(sred1,sz=16)
sred2 = spred(cres1,(16,4,5,7))
cres2 = resiconv(sred2,sz=16)
sred3 = spred(cres2,(16,4,5,7))
sred4 = spred(sred3,(16,4,5,7))
fl1 = Flatten()(sred4)#drop1)
fc1 = Dense(512,kernel_initializer='lecun_normal')(fl1)
drop2 = AlphaDropout(dropout)(fc1)
output = Dense(outputs,kernel_initializer='zeros')(drop2)
output = Activation('sigmoid')(output)
model = Model(inputs=inputdata,outputs=output)
print model.summary()
return model
def init_model(self, features: int = None, batch_size: int = None):
features = len(self.regressors) or features or config.keras.features
batch_size = batch_size or config.keras.batch_size
inp = Input(shape=(self.x_window_size, features),
batch_shape=(batch_size, self.x_window_size, features))
lstm = LSTM(6,
activation='selu',
kernel_initializer='lecun_normal',
stateful=True,
unroll=True)(inp)
dropout = AlphaDropout(0.4)(lstm)
output = Dense(self.y_window_size)(dropout)
self.model = Model(inputs=inp, outputs=output)
self.model.compile(loss='mean_squared_logarithmic_error',
optimizer='nadam',
metrics=['accuracy'])
def create_locally_connected_network(input_size, units, output_size, final_activation, verbose, loss_func = 'categorical_crossentropy', metrics=[], optimizer='adam', inner_local_units_size=50):
# be nice to other processes that also use gpu memory by not monopolizing it on process start
# in case of vram memory limitations on large networks it may be helpful to not set allow_growth and grab it all directly
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session, get_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
from keras.layers import Input, Dense, Activation, Dropout, BatchNormalization, LocallyConnected1D, ZeroPadding1D, Reshape, Flatten
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers.noise import AlphaDropout
from keras.models import Model
kinit = "lecun_normal";
model = Sequential()
model.add(Reshape((input_size, 1), input_shape=(input_size,)))
dropoutRate = 0.1
stride = 500 #this cannot just be changed without fixing padsNeeded to be more general
padsNeeded = ((math.ceil(input_size / stride) + 1) * stride - input_size - 1) % stride
model.add(ZeroPadding1D(padding=(0, padsNeeded)))
model.add(LocallyConnected1D(units[0], 1000, strides=stride, kernel_initializer=kinit))
model.add(Activation("selu"))
for u_cnt in units[1:]:
model.add(LocallyConnected1D(u_cnt, inner_local_units_size, strides=1, kernel_initializer=kinit))
model.add(Activation("selu"))
model.add(AlphaDropout(dropoutRate))
model.add(Flatten())
model.add(Dense(output_size, activation=final_activation, kernel_initializer=kinit))
model.compile(loss=loss_func,
optimizer=optimizer,
metrics=metrics)
if verbose:
model.summary()
return model
def getModel(outputs=2, sz=64,dropout=0.05):
inputdata = Input((1,sz,sz,sz))
c1 = conv3d(sz,kernel_size=3)(inputdata)
red1 = reda(c1)
res1 = iresa(red1,sz=32)
c2 = conv3d(int(res1.shape[1])//2,kernel_size=1)(res1)
res2 = iresa(c2,sz=32)
red2 = redb(res2,(16,5,7,8))
res3 = iresa(red2,sz=32)
c3 = conv3d(int(res3.shape[1])//2,kernel_size=1)(res3)
res4 = iresa(c3,sz=32)
c4 = conv3d(int(res4.shape[1])//2,kernel_size=1)(res4)
fl1 = Flatten()(c4)
fc1 = Dense(128,kernel_initializer='lecun_normal')(fl1)
fc1 = Activation('selu')(fc1)
drop2 = AlphaDropout(dropout)(fc1)
output = Dense(outputs,kernel_initializer='zeros')(drop2)#(fc1)
output = Activation('softmax')(output)
model = Model(inputs=inputdata,outputs=output)
print model.summary()
return model
def get_model(input_shape):
input_features = Input(shape=input_shape,
dtype='float',
name='attribute_features')
from keras import regularizers
from keras.layers.noise import AlphaDropout
from keras.layers import Dropout
x = input_features
# x = keras.layers.BatchNormalization()(x)
x = Dense(
128,
activation='selu',
name='dense_1',
bias_initializer='lecun_normal',
kernel_initializer='lecun_normal',
)(x)
x = AlphaDropout(0.25)(x)
domain = Dense(N_CLS_DOMAIN,
activation='softmax',
name='domain',
bias_regularizer=regularizers.l2(0.1),
kernel_regularizer=regularizers.l2(0.1))(x)
x = keras.layers.concatenate([x, domain])
relation = Dense(N_CLS_RELATION,
activation='softmax',
name='relation',
bias_regularizer=regularizers.l2(0.1),
kernel_regularizer=regularizers.l2(0.1))(x)
clf = Model(inputs=[input_features], outputs=[domain, relation])
return clf
def get_cnn_model(no_of_labels,
X_train,
embedding_dims,
char_to_idx,
filter_tuple,
dropout=0.25,
activation_value='relu',
weights=None):
# activation_value must be relu or tanh
main_input = Input(shape=(X_train.shape[1], ),
dtype='int32',
name='main_input')
print(len(char_to_idx))
print("!!!!!\n\n")
x = Embedding(len(char_to_idx),
embedding_dims,
weights=weights,
input_length=X_train.shape[1])(main_input)
if dropout > 0:
x = Dropout(dropout)(x)
list_cnn_nodes = get_conv_layer(x, filter_tuple, activation_value)
if len(list_cnn_nodes) > 1:
list_cnn_nodes = layers.Concatenate()(list_cnn_nodes)
else:
list_cnn_nodes = list_cnn_nodes[0]
# for i in range(len(list_cnn_nodes)):
# print(list_cnn_nodes[i])
print("list_cnn_nodes\n")
# Att = AttLayer()(list_cnn_nodes)
if dropout > 0:
list_cnn_nodes = AlphaDropout(dropout)(list_cnn_nodes)
main_loss = Dense(no_of_labels, activation='softmax',
name='main_output')(list_cnn_nodes)
print("here\n\n\n")
model = Model(input=main_input, output=main_loss)
print("OK\n\n\n")
return model
# model = Sequential()
inputs = Input(shape=(32, 32, 3))
x = Conv2D(32, (3, 3),
padding='same',
input_shape=x_train.shape[1:],
kernel_initializer='lecun_normal')(inputs)
# model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))
x = Activation('selu')(x)
# model.add(Activation('relu'))
x = Conv2D(32, (3, 3), kernel_initializer='lecun_normal')(x)
# model.add(Conv2D(32, (3, 3)))
x = Activation('selu')(x)
# model.add(Activation('relu'))
x = MaxPooling2D(pool_size=(2, 2))(x)
# model.add(MaxPooling2D(pool_size=(2, 2)))
x = AlphaDropout(0.25)(x)
# model.add(Dropout(0.25))
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='lecun_normal')(x)
# model.add(Conv2D(64, (3, 3), padding='same'))
x = Activation('selu')(x)
# model.add(Activation('relu'))
x = Conv2D(64, (3, 3), kernel_initializer='lecun_normal')(x)
# model.add(Conv2D(64, (3, 3)))
x = Activation('selu')(x)
# model.add(Activation('relu'))
x = MaxPooling2D(pool_size=(2, 2))(x)
# model.add(MaxPooling2D(pool_size=(2, 2)))
x = AlphaDropout(0.25)(x)
# model.add(Dropout(0.25))
x = Flatten()(x)
# model.add(Flatten())
# maxpool for down sampling
x = layers.MaxPooling1D(data_format='channels_first')(x)
return x
inputs = layers.Input(shape=list(x_train.shape[1:]))
x = residual_stack(inputs)
x = residual_stack(x)
#x = residual_stack(x) # Comment this when the input dimensions are 1/32 or lower
#x = residual_stack(x) # Comment this when the input dimensions are 1/16 or lower
#x = residual_stack(x) # Comment this when the input dimensions are 1/8 or lower
x = Flatten()(x)
x = Dense(128,
kernel_initializer="he_normal",
activation="selu",
name="dense1")(x)
x = AlphaDropout(0.1)(x)
x = Dense(128,
kernel_initializer="he_normal",
activation="selu",
name="dense2")(x)
x = AlphaDropout(0.1)(x)
x = Dense(len(classes),
kernel_initializer="he_normal",
activation="softmax",
name="dense3")(x)
x_out = Reshape([len(classes)])(x)
model = models.Model(inputs=[inputs], output=[x_out])
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()
# Set up some params
nb_epoch = 500 # number of epochs to train on
def train(self,
training_file,
architecture="cnn",
window_size=4,
epochs=5,
batch_size=32,
dropout=0.25,
min_freq=10000,
max_features=20000,
embedding_size=128,
lstm_gru_size=256,
mlp_dense=6,
mlp_dense_units=16,
kernel_size=5,
filters=64,
pool_size=2,
hidden_dims=250,
strides=1,
model_filename='best_model.hdf5',
vocab_filename='vocab.dump'):
with open(training_file, mode='r', encoding='utf-8') as f:
training_corpus = f.read()
data_set_char = self.util.build_data_set_char(training_corpus,
window_size)
char_2_id_dict = self.util.build_char_2_id_dict(
data_set_char, min_freq)
data_set = self.util.build_data_set(data_set_char, char_2_id_dict,
window_size)
x_train = np.array([i[1] for i in data_set])
y_train = np.array([i[0] for i in data_set])
maxlen = 2 * window_size + 1
model = Sequential()
if architecture == "cnn":
model.add(
Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(dropout))
# we add a Convolution1D, which will learn filters
# word group filters of size filter_length:
model.add(
Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1))
# we use max pooling:
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(dropout))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a
# sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
elif architecture == "lstm":
model.add(Embedding(max_features, embedding_size))
model.add(
LSTM(lstm_gru_size, dropout=dropout,
recurrent_dropout=dropout))
model.add(Dense(1, activation='sigmoid'))
elif architecture == "bi-lstm":
model.add(Embedding(max_features, embedding_size))
model.add(
Bidirectional(
LSTM(lstm_gru_size,
dropout=dropout,
recurrent_dropout=dropout)))
model.add(Dense(1, activation='sigmoid'))
elif architecture == "gru":
model.add(Embedding(max_features, embedding_size))
model.add(
GRU(lstm_gru_size, dropout=dropout, recurrent_dropout=dropout))
model.add(Dense(1, activation='sigmoid'))
elif architecture == "bi-gru":
model.add(Embedding(max_features, embedding_size))
model.add(
Bidirectional(
GRU(lstm_gru_size,
dropout=dropout,
recurrent_dropout=dropout)))
model.add(Dense(1, activation='sigmoid'))
elif architecture == "mlp":
model.add(
Dense(mlp_dense_units,
input_shape=(maxlen, ),
kernel_initializer='lecun_normal'))
model.add(Activation('selu'))
model.add(AlphaDropout(dropout))
for i in range(mlp_dense - 1):
model.add(
Dense(mlp_dense_units, kernel_initializer='lecun_normal'))
model.add(Activation('selu'))
model.add(AlphaDropout(dropout))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
plot_model(model, to_file='model.png')
mcp = ModelCheckpoint(model_filename,
monitor="acc",
save_best_only=True,
save_weights_only=False,
mode='max')
model.fit(x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
callbacks=[mcp])
self.util.save_vocab(char_2_id_dict, vocab_filename)
# GRU output for loss calculation #1
gru_pred = Dense(1, name='GRU_out')(gru_out)
# merge other data with GRU output
aux_input = Input(shape=X2.shape[1:], name='aux_in')
x = concatenate([gru_out, aux_input])
activation = 'selu'
layer_size = 128
n_layers = 2
drop_rate = 0.01
for n in range(n_layers):
x = Dense(layer_size)(x)
x = Activation(activation)(x)
x = AlphaDropout(rate=drop_rate)(x)
# main output for loss calculation #2
main_pred = Dense(1, name='main_out')(x)
# define inputs / outputs
model = Model(inputs=[main_input, aux_input], outputs=[main_pred, gru_pred])
# weight losses and compile model
model.compile(optimizer=TFOptimizer(YFOptimizer()),
loss='mean_squared_error',
loss_weights=[.5, .5])
model.fit([X1, X2], [y, y],
epochs=50,
validation_split=0.5,
raw_data[filtered_columns.columns.tolist()])
# Convert labels from binary classes to one-hot vector
labels = to_categorical(raw_data['label'])
# Split data
train_data, test_data, train_labels, test_labels = train_test_split(
data, labels, test_size=0.1)
# Prepare TensorBoard
tb = TensorBoard(log_dir='logs', histogram_freq=0, write_graph=True)
model = Sequential()
model.add(Dense(64, input_shape=(169, ), activation='selu'))
model.add(AlphaDropout(dr))
model.add(Dense(128, activation='selu'))
model.add(AlphaDropout(dr))
# model.add(Dense(128, activation='selu'))
# model.add(Dense(256, activation='selu'))
# model.add(Dense(256, activation='selu'))
# model.add(Dense(128, activation='selu'))
model.add(Dense(128, activation='selu'))
model.add(AlphaDropout(dr))
model.add(Dense(64, activation='selu'))
model.add(AlphaDropout(dr))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer=Adam(lr=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
def Unet(img_size):
inputs = Input((img_size, img_size, 3))
s = Lambda(lambda x: x / 255)(inputs)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(s)
c1 = AlphaDropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = AlphaDropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = AlphaDropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = AlphaDropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = AlphaDropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = AlphaDropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = AlphaDropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = AlphaDropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = AlphaDropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
return model
#classifier.add(AlphaDropout(0.1))
# Adding a fourth convolutional layer
classifier.add(
Conv2D(256, (3, 3), kernel_initializer='lecun_normal', padding='same'))
classifier.add(Activation('selu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))
#classifier.add(AlphaDropout(0.05))
# Step 3 - Flattening
classifier.add(Flatten())
# Step 4 - Full connection
classifier.add(Dense(units=128, kernel_initializer='lecun_normal'))
classifier.add(Activation('selu'))
classifier.add(AlphaDropout(0.2))
classifier.add(Dense(units=256, kernel_initializer='lecun_normal'))
classifier.add(Activation('selu'))
classifier.add(AlphaDropout(0.2))
#output layer
classifier.add(Dense(units=1, activation='sigmoid'))
# Compiling the CNN
classifier.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
# Part 2 - Fitting the CNN to the images
def main(
data=None,
params={
'lr': 0.001,
'conv_dr': 0.,
'fc_dr': 0.1,
'batch_size': 128,
'no_epochs': 1000,
'steps_per_epoch': 100,
'dp_prob': 0.5,
'batch_norm': False,
'regularise': 0.0,
'decay': 0.0
},
no_threads=10,
verbose=True,
cp_interval=100,
test=False):
"""
Runs a self-normalising neural network on the waveform data, saving the model to the filestore
Requires a directory called 'logs' in the same folder for the TensorBoard visualisation
The current neural network structure is:
fc > dropout > fc > dropout > fc > dropout > fc > dropout > softmax
Arguments:
data - the Datasets object to run on, if None then loads data (default = None)
params - a dictionary object containing the following parameters
lr - the learning rate of the Adam optimiser (default = 0.001)
conv_dr - unused
fc_dr - the dropout rate for the alpha dropout layers (default = 0.1)
no_epochs - the number of epochs to run for
steps_per_epoch - the number of batches in each epoch
dp_prob - the proportion of double pulse waveforms shown at train time (default = 0.5)
batch_norm - unused
regularise - sets the amount of L2 regularisation for each layer (default = 0.0)
decay - sets the decay rate for the proportion of double-pulse waveforms used for
training and validation (default = 0.0)
no_threads - number of threads to use (default is 10, use -1 to set no limit)
verbose - dictates the amount of output that keras gives
cp_interval - the number of epochs between saving model checkpoints (default = 100)
test - suppresses saving of model and output of logs (for testing new features; default = False)
No returns
"""
# Read in data
if data == None:
data = load_data(verbose=verbose)
# Set up CPU, GPU options
config = None
if no_threads == -1:
config = tf.ConfigProto(allow_soft_placement=True,
device_count={
'CPU': 1,
'GPU': 1
},
gpu_options=tf.GPUOptions(allow_growth=True))
else:
config = tf.ConfigProto(intra_op_parallelism_threads=no_threads,
inter_op_parallelism_threads=no_threads,
allow_soft_placement=True,
device_count={
'CPU': 1,
'GPU': 1
},
gpu_options=tf.GPUOptions(allow_growth=True))
sess = tf.Session(config=config)
K.set_session(sess)
# Define model
model = Sequential()
# Set up regulariser
regulariser = l2(params['regularise'])
model.add(
Dense(1024,
input_shape=(128, ),
activation='selu',
kernel_regularizer=regulariser))
model.add(AlphaDropout(params['fc_dr']))
model.add(Dense(1024, activation='selu', kernel_regularizer=regulariser))
model.add(AlphaDropout(params['fc_dr']))
model.add(Dense(1024, activation='selu', kernel_regularizer=regulariser))
model.add(AlphaDropout(params['fc_dr']))
model.add(Dense(64, activation='selu', kernel_regularizer=regulariser))
model.add(AlphaDropout(params['fc_dr']))
model.add(Dense(2, activation='softmax'))
# Set-up optimiser
optimiser = Adam(lr=params['lr'])
# Create model
model.compile(optimizer=optimiser,
loss='categorical_crossentropy',
metrics=['accuracy', precision, recall, f1, class_balance])
# Create generators for training, validation
train_gen = WaveformGenerator(data.train,
batch_size=params['batch_size'],
balanced=True,
dp_prob=params['dp_prob'],
decay=params['decay'])
val_gen = WaveformGenerator(data.val,
batch_size=params['batch_size'],
balanced=True,
dp_prob=params['dp_prob'],
decay=params['decay'])
# Prepare callbacks
callbacks = [train_gen, val_gen]
if test == False:
tb = TensorBoard(log_dir='logs', histogram_freq=0, write_graph=True)
model_saver = ModelSaver(model,
'snn',
params,
verbose=verbose,
period=cp_interval)
callbacks += [tb, model_saver]
# Train model
model.fit_generator(train_gen,
steps_per_epoch=params['steps_per_epoch'],
epochs=params['no_epochs'],
verbose=int(verbose),
validation_data=val_gen,
validation_steps=params['steps_per_epoch'],
callbacks=callbacks)
# Evaluate model
test_preds = model.predict(data.val.waveforms, verbose=int(verbose))
print()
print_metric_results(data.val.labels,
test_preds,
data.val.weights,
data.val.ids,
th=0.5)
print_metric_results(data.val.labels,
test_preds,
data.val.weights,
data.val.ids,
th=0.9)
def getModel():
inputdata = Input((1, 64, 64, 64))
conv1 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(inputdata)
conv1 = PReLU()(conv1)
conv2 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(conv1)
conv2 = PReLU()(conv2)
conv3 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(conv2)
conv3 = PReLU()(conv3)
pool1 = MaxPooling3D(pool_size=(2, 2, 2),
data_format='channels_first')(conv3)
encode1 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(pool1)
encode1 = PReLU()(encode1)
encode2 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(encode1)
encode2 = PReLU()(encode2)
encode3 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(encode2)
encode3 = PReLU()(encode3)
encode4 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(encode3)
encode4 = PReLU()(encode4)
upsampling1 = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(encode4)
finalShape = upsampling1.shape
originalShape = conv3.shape
cropShape = int(originalShape[2]/2-finalShape[2]/2),int(originalShape[3]/2-\
finalShape[3]/2),int(originalShape[4]/2-finalShape[4]/2)
crop1 = Cropping3D(cropping=cropShape, data_format='channels_first')(conv3)
concatenate1 = concatenate([upsampling1, crop1], axis=1)
dropout1 = AlphaDropout(0.1)(concatenate1)
expand1 = Convolution3D(256,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(dropout1)
expand1 = PReLU()(expand1)
expand2 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(expand1)
expand2 = PReLU()(expand2)
expand3 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(expand2)
expand3 = PReLU()(expand3)
expand4 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(expand3)
expand4 = PReLU()(expand4)
expand5 = Convolution3D(128,
kernel_size=3,
data_format='channels_first',
activation='selu',
padding='valid',
use_bias=True,
kernel_initializer='lecun_normal')(expand4)
expand5 = PReLU()(expand5)
outputdata = Convolution3D(1,
kernel_size=1,
data_format='channels_first',
activation='sigmoid',
padding='valid',
use_bias=True,
kernel_initializer='glorot_normal')(expand5)
model = Model(inputs=inputdata, outputs=outputdata)
model.compile(optimizer=Adam(lr=LR), loss=diceCoef)
print model.summary()
return model
# Hyperparameters
EPOCHS = 6
BATCH_SIZE = 32
LEARNING_RATE = 1.0e-3
# Data
train_generator = generator(train_samples, batch_size=BATCH_SIZE)
valid_generator = generator(valid_samples, batch_size=BATCH_SIZE)
# Define the model
#model = load_model("./model.h5")
model = Sequential()
model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=SHAPE))
model.add(
Conv2D(2, (3, 3), activation='selu', kernel_initializer='lecun_normal'))
model.add(MaxPooling2D((4, 4)))
model.add(AlphaDropout(0.25))
model.add(Flatten())
model.add(Dense(1))
model.compile(loss='mse', optimizer=optimizers.Adam(lr=LEARNING_RATE))
# Train the model
model.fit_generator(train_generator,
steps_per_epoch=np.ceil(len(train_samples) / BATCH_SIZE),
validation_data=valid_generator,
validation_steps=np.ceil(len(train_samples) / BATCH_SIZE),
epochs=EPOCHS)
# Save the model
model.save("./model.h5")
def resnet(X_train, X_test):
Y_train = to_onehot(
map(lambda x: mods.index(lbl[x][0]), range(X_train.shape[0])))
Y_test = to_onehot(
map(lambda x: mods.index(lbl[x][0]), range(X_test.shape[0])))
print('training started')
in_shp = list(X_train.shape[1:])
print(X_train.shape, in_shp)
classes = mods
# Resnet Architecture
def residual_stack(x):
def residual_unit(y, _strides=1):
shortcut_unit = y
# 1x1 conv linear
y = layers.Conv1D(32,
kernel_size=5,
data_format='channels_first',
strides=_strides,
padding='same',
activation='relu')(y)
y = layers.BatchNormalization()(y)
y = layers.Conv1D(32,
kernel_size=5,
data_format='channels_first',
strides=_strides,
padding='same',
activation='linear')(y)
y = layers.BatchNormalization()(y)
# add batch normalization
y = layers.add([shortcut_unit, y])
return y
x = layers.Conv1D(32,
data_format='channels_first',
kernel_size=1,
padding='same',
activation='linear')(x)
x = layers.BatchNormalization()(x)
x = residual_unit(x)
x = residual_unit(x)
# maxpool for down sampling
x = layers.MaxPooling1D(data_format='channels_first')(x)
return x
inputs = layers.Input(shape=in_shp)
x = residual_stack(inputs)
x = residual_stack(x)
x = residual_stack(
x) # Comment this when the input dimensions are 1/32 or lower
x = residual_stack(
x) # Comment this when the input dimensions are 1/16 or lower
x = residual_stack(
x) # Comment this when the input dimensions are 1/8 or lower
x = Flatten()(x)
x = Dense(128,
kernel_initializer="he_normal",
activation="selu",
name="dense1")(x)
x = AlphaDropout(0.1)(x)
x = Dense(128,
kernel_initializer="he_normal",
activation="selu",
name="dense2")(x)
x = AlphaDropout(0.1)(x)
x = Dense(len(classes),
kernel_initializer="he_normal",
activation="softmax",
name="dense3")(x)
x_out = Reshape([len(classes)])(x)
model = models.Model(inputs=[inputs], output=[x_out])
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()
# Set up some params
nb_epoch = 500 # number of epochs to train on
batch_size = 1024 # training batch size
# Train the Model
filepath = 'models/weights_resnet.wts.h5'
model = multi_gpu_model(model, gpus=3)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.001),
metrics=['accuracy'])
history = model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
verbose=2,
validation_split=0.25,
callbacks=[
keras.callbacks.ModelCheckpoint(
filepath,
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='auto'),
keras.callbacks.EarlyStopping(monitor='val_loss',
patience=10,
verbose=0,
mode='auto')
])
# we re-load the best weights once training is finished
model.load_weights(filepath)
score = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=0)
return score[1]
def train_and_test(X_train_word,
X_train_syntax,
Y_train,
X_test_word,
X_test_syntax,
Y_test,
nb_epochs,
batch_size,
learning_rate,
no_of_labels,
embedding_dims_word,
embedding_dims_syntax,
char_to_idx,
filter_tuple,
filter_tuple1,
dropout,
activation_value,
length,
label_dic,
model_type,
logdir,
weights=None,
weights_pos=None):
# activation_value must be relu or tanh
bm_scores = True
binary = False
main_input = Input(shape=(X_train_word.shape[1], ),
dtype='int32',
name='main_input')
x = Embedding(len(char_to_idx),
embedding_dims_word,
weights=weights,
input_length=X_train_word.shape[1])(main_input)
if dropout > 0:
x = Dropout(dropout)(x)
list_cnn_nodes = get_conv_layer(x, filter_tuple, activation_value)
if len(list_cnn_nodes) > 1:
list_cnn_nodes = layers.Concatenate()(list_cnn_nodes)
else:
list_cnn_nodes = list_cnn_nodes[0]
if dropout > 0:
list_cnn_nodes = AlphaDropout(dropout)(list_cnn_nodes)
# the start of syntax ----------------
main_input_syntax = Input(shape=(X_train_syntax.shape[1],
X_train_syntax.shape[2]),
dtype='int32',
name='main_input_syntax')
x_syntax = Syntax_Embedding(input_dim=label_dic,
syntax_tree_label_dim=embedding_dims_syntax,
depth_of_syntax_tree=150,
input_length=length)(main_input_syntax)
if dropout > 0:
x_syntax = Dropout(dropout)(x_syntax)
list_cnn_nodes_syntax = get_conv_layer_syntax(x_syntax, filter_tuple1,
activation_value)
if len(list_cnn_nodes_syntax) > 1:
list_cnn_nodes_syntax = layers.Concatenate()(list_cnn_nodes_syntax)
else:
list_cnn_nodes_syntax = list_cnn_nodes_syntax[0]
if dropout > 0:
list_cnn_nodes_syntax = AlphaDropout(dropout)(list_cnn_nodes_syntax)
# the end of syntax -----------------
###how to concat
list_final = []
list_final.append(list_cnn_nodes)
list_final.append(list_cnn_nodes_syntax)
list_cnn_nodes = layers.Concatenate()(list_final)
main_loss = Dense(no_of_labels, activation='softmax',
name='main_output')(list_cnn_nodes)
model = Model(inputs=[main_input, main_input_syntax], output=main_loss)
print("model")
timestamp = time.strftime("%Y%m%d-%H%M%S")
# checkpointer = ModelCheckpoint(save_best_only=True, monitor='val_acc', verbose=1, filepath=os.path.join(logdir, model_type+timestamp+".hdf5"), )
# dict_callbacks = {'checkpointer':checkpointer}
# , callbacks=dict_callbacks.values()
adam = Adam(lr=learning_rate)
model.summary()
if binary:
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
else:
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
history = model.fit([X_train_word, X_train_syntax],
Y_train,
batch_size=batch_size,
nb_epoch=300,
validation_data=([X_test_word, X_test_syntax], Y_test))
open(os.path.join(logdir, model_type + timestamp + ".json"),
'w').write(model.to_json())
model.summary()
Y_TRAIN = to_categorical(Y_TRAIN, N_CLASSES)
Y_TEST = to_categorical(Y_TEST, N_CLASSES)
print('X_TRAIN shape --> {0}'.format(X_TRAIN.shape))
print('Y_TRAIN shape --> {0}'.format(Y_TRAIN.shape))
print('X_TEST shape --> {0}'.format(X_TEST.shape))
print('Y_TEST shape --> {0}'.format(Y_TEST.shape))
IMG_SHAPE = X_TRAIN[0].shape
MODEL = Sequential()
MODEL.add(InputLayer(input_shape=IMG_SHAPE))
MODEL.add(Dense(512))
MODEL.add(Activation('selu'))
MODEL.add(AlphaDropout(0.2))
MODEL.add(Dense(512))
MODEL.add(Activation('selu'))
MODEL.add(AlphaDropout(0.2))
MODEL.add(Dense(N_CLASSES))
MODEL.add(Activation('softmax'))
MODEL.summary()
plot_model(MODEL, to_file='../bin/snn_mnist_2_hidden.png')
with open('../bin/snn_mnist_2_hidden.yaml', 'w') as f:
f.write(MODEL.to_yaml())
MODEL.compile(loss=categorical_crossentropy,
optimizer=RMSprop(),