def single_nn(opt, dropout, a_func, reg_all, preds_reg, embed_dim, dense_units): user_embeddings = Embedding(u_cnt, embed_dim, embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1), embeddings_regularizer=l2(reg_all), input_length=1, trainable=True, name='user_embeddings') song_embeddings = Embedding(s_cnt, embed_dim, embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1), embeddings_regularizer=l2(reg_all), input_length=1, trainable=True,name='item_embeddings') uid_input = Input(shape=(1,), dtype='int32') embedded_usr = user_embeddings(uid_input) embedded_usr = Reshape((embed_dim,),name='user_embeddings_reshaped')(embedded_usr) ub = create_bias(uid_input, u_cnt, reg_all,'user_biases') sid_input = Input(shape=(1,), dtype='int32') embedded_song = song_embeddings(sid_input) embedded_song = Reshape((embed_dim,),name='item_embeddings_reshaped')(embedded_song) mb = create_bias(sid_input, s_cnt, reg_all,'item_biases') preds = dot([embedded_usr, embedded_song], axes=1) preds = concatenate([embedded_usr, embedded_song, preds], name='concatenated_embeddings_all') if (a_func == 'relu') or (a_func == 'elu'): preds = Dense(1, use_bias=True, activation=a_func, kernel_initializer=RandomNormal(mean=0.0, stddev=0.1), kernel_regularizer=regularizers.l2(preds_reg), bias_initializer=RandomNormal(mean=0.0, stddev=0.1),bias_regularizer=regularizers.l2(preds_reg), name='main_output')(preds) preds=Dropout(dropout)(preds) elif a_func == 'selu': preds = Dense(1, activation=a_func, kernel_initializer=lecun_normal(), name='main_output')(preds) preds=AlphaDropout(dropout)(preds) preds = add([ub, preds]) preds = add([mb, preds]) model = Model(inputs=[uid_input, sid_input], outputs=preds) opt = eval(opt) model.compile(loss='mse', optimizer=opt, metrics=[mae]) return model
def getModel(outputs=2, sz=32, dropout=0.05, full=True): inputdata = Input((1,32,32,32)) c1 = conv3d(sz,kernel_size=3)(inputdata) sred1 = spred(c1,(16,4,5,7)) cres1 = resiconv(sred1,sz=16) sred2 = spred(cres1,(16,4,5,7)) cres2 = resiconv(sred2,sz=16) sred3 = spred(cres2,(16,4,5,7)) sred4 = spred(sred3,(16,4,5,7)) fl1 = Flatten()(sred4)#drop1) fc1 = Dense(512,kernel_initializer='lecun_normal')(fl1) drop2 = AlphaDropout(dropout)(fc1) output = Dense(outputs,kernel_initializer='zeros')(drop2) output = Activation('sigmoid')(output) model = Model(inputs=inputdata,outputs=output) print model.summary() return model
def init_model(self, features: int = None, batch_size: int = None): features = len(self.regressors) or features or config.keras.features batch_size = batch_size or config.keras.batch_size inp = Input(shape=(self.x_window_size, features), batch_shape=(batch_size, self.x_window_size, features)) lstm = LSTM(6, activation='selu', kernel_initializer='lecun_normal', stateful=True, unroll=True)(inp) dropout = AlphaDropout(0.4)(lstm) output = Dense(self.y_window_size)(dropout) self.model = Model(inputs=inp, outputs=output) self.model.compile(loss='mean_squared_logarithmic_error', optimizer='nadam', metrics=['accuracy'])
def create_locally_connected_network(input_size, units, output_size, final_activation, verbose, loss_func = 'categorical_crossentropy', metrics=[], optimizer='adam', inner_local_units_size=50): # be nice to other processes that also use gpu memory by not monopolizing it on process start # in case of vram memory limitations on large networks it may be helpful to not set allow_growth and grab it all directly import tensorflow as tf from keras.backend.tensorflow_backend import set_session, get_session config = tf.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.Session(config=config)) from keras.layers import Input, Dense, Activation, Dropout, BatchNormalization, LocallyConnected1D, ZeroPadding1D, Reshape, Flatten from keras.models import Sequential from keras.optimizers import Adam from keras.layers.noise import AlphaDropout from keras.models import Model kinit = "lecun_normal"; model = Sequential() model.add(Reshape((input_size, 1), input_shape=(input_size,))) dropoutRate = 0.1 stride = 500 #this cannot just be changed without fixing padsNeeded to be more general padsNeeded = ((math.ceil(input_size / stride) + 1) * stride - input_size - 1) % stride model.add(ZeroPadding1D(padding=(0, padsNeeded))) model.add(LocallyConnected1D(units[0], 1000, strides=stride, kernel_initializer=kinit)) model.add(Activation("selu")) for u_cnt in units[1:]: model.add(LocallyConnected1D(u_cnt, inner_local_units_size, strides=1, kernel_initializer=kinit)) model.add(Activation("selu")) model.add(AlphaDropout(dropoutRate)) model.add(Flatten()) model.add(Dense(output_size, activation=final_activation, kernel_initializer=kinit)) model.compile(loss=loss_func, optimizer=optimizer, metrics=metrics) if verbose: model.summary() return model
def getModel(outputs=2, sz=64,dropout=0.05): inputdata = Input((1,sz,sz,sz)) c1 = conv3d(sz,kernel_size=3)(inputdata) red1 = reda(c1) res1 = iresa(red1,sz=32) c2 = conv3d(int(res1.shape[1])//2,kernel_size=1)(res1) res2 = iresa(c2,sz=32) red2 = redb(res2,(16,5,7,8)) res3 = iresa(red2,sz=32) c3 = conv3d(int(res3.shape[1])//2,kernel_size=1)(res3) res4 = iresa(c3,sz=32) c4 = conv3d(int(res4.shape[1])//2,kernel_size=1)(res4) fl1 = Flatten()(c4) fc1 = Dense(128,kernel_initializer='lecun_normal')(fl1) fc1 = Activation('selu')(fc1) drop2 = AlphaDropout(dropout)(fc1) output = Dense(outputs,kernel_initializer='zeros')(drop2)#(fc1) output = Activation('softmax')(output) model = Model(inputs=inputdata,outputs=output) print model.summary() return model
def get_model(input_shape): input_features = Input(shape=input_shape, dtype='float', name='attribute_features') from keras import regularizers from keras.layers.noise import AlphaDropout from keras.layers import Dropout x = input_features # x = keras.layers.BatchNormalization()(x) x = Dense( 128, activation='selu', name='dense_1', bias_initializer='lecun_normal', kernel_initializer='lecun_normal', )(x) x = AlphaDropout(0.25)(x) domain = Dense(N_CLS_DOMAIN, activation='softmax', name='domain', bias_regularizer=regularizers.l2(0.1), kernel_regularizer=regularizers.l2(0.1))(x) x = keras.layers.concatenate([x, domain]) relation = Dense(N_CLS_RELATION, activation='softmax', name='relation', bias_regularizer=regularizers.l2(0.1), kernel_regularizer=regularizers.l2(0.1))(x) clf = Model(inputs=[input_features], outputs=[domain, relation]) return clf
def get_cnn_model(no_of_labels, X_train, embedding_dims, char_to_idx, filter_tuple, dropout=0.25, activation_value='relu', weights=None): # activation_value must be relu or tanh main_input = Input(shape=(X_train.shape[1], ), dtype='int32', name='main_input') print(len(char_to_idx)) print("!!!!!\n\n") x = Embedding(len(char_to_idx), embedding_dims, weights=weights, input_length=X_train.shape[1])(main_input) if dropout > 0: x = Dropout(dropout)(x) list_cnn_nodes = get_conv_layer(x, filter_tuple, activation_value) if len(list_cnn_nodes) > 1: list_cnn_nodes = layers.Concatenate()(list_cnn_nodes) else: list_cnn_nodes = list_cnn_nodes[0] # for i in range(len(list_cnn_nodes)): # print(list_cnn_nodes[i]) print("list_cnn_nodes\n") # Att = AttLayer()(list_cnn_nodes) if dropout > 0: list_cnn_nodes = AlphaDropout(dropout)(list_cnn_nodes) main_loss = Dense(no_of_labels, activation='softmax', name='main_output')(list_cnn_nodes) print("here\n\n\n") model = Model(input=main_input, output=main_loss) print("OK\n\n\n") return model
# model = Sequential() inputs = Input(shape=(32, 32, 3)) x = Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:], kernel_initializer='lecun_normal')(inputs) # model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:])) x = Activation('selu')(x) # model.add(Activation('relu')) x = Conv2D(32, (3, 3), kernel_initializer='lecun_normal')(x) # model.add(Conv2D(32, (3, 3))) x = Activation('selu')(x) # model.add(Activation('relu')) x = MaxPooling2D(pool_size=(2, 2))(x) # model.add(MaxPooling2D(pool_size=(2, 2))) x = AlphaDropout(0.25)(x) # model.add(Dropout(0.25)) x = Conv2D(64, (3, 3), padding='same', kernel_initializer='lecun_normal')(x) # model.add(Conv2D(64, (3, 3), padding='same')) x = Activation('selu')(x) # model.add(Activation('relu')) x = Conv2D(64, (3, 3), kernel_initializer='lecun_normal')(x) # model.add(Conv2D(64, (3, 3))) x = Activation('selu')(x) # model.add(Activation('relu')) x = MaxPooling2D(pool_size=(2, 2))(x) # model.add(MaxPooling2D(pool_size=(2, 2))) x = AlphaDropout(0.25)(x) # model.add(Dropout(0.25)) x = Flatten()(x) # model.add(Flatten())
# maxpool for down sampling x = layers.MaxPooling1D(data_format='channels_first')(x) return x inputs = layers.Input(shape=list(x_train.shape[1:])) x = residual_stack(inputs) x = residual_stack(x) #x = residual_stack(x) # Comment this when the input dimensions are 1/32 or lower #x = residual_stack(x) # Comment this when the input dimensions are 1/16 or lower #x = residual_stack(x) # Comment this when the input dimensions are 1/8 or lower x = Flatten()(x) x = Dense(128, kernel_initializer="he_normal", activation="selu", name="dense1")(x) x = AlphaDropout(0.1)(x) x = Dense(128, kernel_initializer="he_normal", activation="selu", name="dense2")(x) x = AlphaDropout(0.1)(x) x = Dense(len(classes), kernel_initializer="he_normal", activation="softmax", name="dense3")(x) x_out = Reshape([len(classes)])(x) model = models.Model(inputs=[inputs], output=[x_out]) model.compile(loss='categorical_crossentropy', optimizer='adam') model.summary() # Set up some params nb_epoch = 500 # number of epochs to train on
def train(self, training_file, architecture="cnn", window_size=4, epochs=5, batch_size=32, dropout=0.25, min_freq=10000, max_features=20000, embedding_size=128, lstm_gru_size=256, mlp_dense=6, mlp_dense_units=16, kernel_size=5, filters=64, pool_size=2, hidden_dims=250, strides=1, model_filename='best_model.hdf5', vocab_filename='vocab.dump'): with open(training_file, mode='r', encoding='utf-8') as f: training_corpus = f.read() data_set_char = self.util.build_data_set_char(training_corpus, window_size) char_2_id_dict = self.util.build_char_2_id_dict( data_set_char, min_freq) data_set = self.util.build_data_set(data_set_char, char_2_id_dict, window_size) x_train = np.array([i[1] for i in data_set]) y_train = np.array([i[0] for i in data_set]) maxlen = 2 * window_size + 1 model = Sequential() if architecture == "cnn": model.add( Embedding(max_features, embedding_size, input_length=maxlen)) model.add(Dropout(dropout)) # we add a Convolution1D, which will learn filters # word group filters of size filter_length: model.add( Conv1D(filters, kernel_size, padding='valid', activation='relu', strides=1)) # we use max pooling: model.add(GlobalMaxPooling1D()) # We add a vanilla hidden layer: model.add(Dense(hidden_dims)) model.add(Dropout(dropout)) model.add(Activation('relu')) # We project onto a single unit output layer, and squash it with a # sigmoid: model.add(Dense(1)) model.add(Activation('sigmoid')) elif architecture == "lstm": model.add(Embedding(max_features, embedding_size)) model.add( LSTM(lstm_gru_size, dropout=dropout, recurrent_dropout=dropout)) model.add(Dense(1, activation='sigmoid')) elif architecture == "bi-lstm": model.add(Embedding(max_features, embedding_size)) model.add( Bidirectional( LSTM(lstm_gru_size, dropout=dropout, recurrent_dropout=dropout))) model.add(Dense(1, activation='sigmoid')) elif architecture == "gru": model.add(Embedding(max_features, embedding_size)) model.add( GRU(lstm_gru_size, dropout=dropout, recurrent_dropout=dropout)) model.add(Dense(1, activation='sigmoid')) elif architecture == "bi-gru": model.add(Embedding(max_features, embedding_size)) model.add( Bidirectional( GRU(lstm_gru_size, dropout=dropout, recurrent_dropout=dropout))) model.add(Dense(1, activation='sigmoid')) elif architecture == "mlp": model.add( Dense(mlp_dense_units, input_shape=(maxlen, ), kernel_initializer='lecun_normal')) model.add(Activation('selu')) model.add(AlphaDropout(dropout)) for i in range(mlp_dense - 1): model.add( Dense(mlp_dense_units, kernel_initializer='lecun_normal')) model.add(Activation('selu')) model.add(AlphaDropout(dropout)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() plot_model(model, to_file='model.png') mcp = ModelCheckpoint(model_filename, monitor="acc", save_best_only=True, save_weights_only=False, mode='max') model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, callbacks=[mcp]) self.util.save_vocab(char_2_id_dict, vocab_filename)
# GRU output for loss calculation #1 gru_pred = Dense(1, name='GRU_out')(gru_out) # merge other data with GRU output aux_input = Input(shape=X2.shape[1:], name='aux_in') x = concatenate([gru_out, aux_input]) activation = 'selu' layer_size = 128 n_layers = 2 drop_rate = 0.01 for n in range(n_layers): x = Dense(layer_size)(x) x = Activation(activation)(x) x = AlphaDropout(rate=drop_rate)(x) # main output for loss calculation #2 main_pred = Dense(1, name='main_out')(x) # define inputs / outputs model = Model(inputs=[main_input, aux_input], outputs=[main_pred, gru_pred]) # weight losses and compile model model.compile(optimizer=TFOptimizer(YFOptimizer()), loss='mean_squared_error', loss_weights=[.5, .5]) model.fit([X1, X2], [y, y], epochs=50, validation_split=0.5,
raw_data[filtered_columns.columns.tolist()]) # Convert labels from binary classes to one-hot vector labels = to_categorical(raw_data['label']) # Split data train_data, test_data, train_labels, test_labels = train_test_split( data, labels, test_size=0.1) # Prepare TensorBoard tb = TensorBoard(log_dir='logs', histogram_freq=0, write_graph=True) model = Sequential() model.add(Dense(64, input_shape=(169, ), activation='selu')) model.add(AlphaDropout(dr)) model.add(Dense(128, activation='selu')) model.add(AlphaDropout(dr)) # model.add(Dense(128, activation='selu')) # model.add(Dense(256, activation='selu')) # model.add(Dense(256, activation='selu')) # model.add(Dense(128, activation='selu')) model.add(Dense(128, activation='selu')) model.add(AlphaDropout(dr)) model.add(Dense(64, activation='selu')) model.add(AlphaDropout(dr)) model.add(Dense(2, activation='softmax')) model.compile(optimizer=Adam(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
def Unet(img_size): inputs = Input((img_size, img_size, 3)) s = Lambda(lambda x: x / 255)(inputs) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(s) c1 = AlphaDropout(0.1)(c1) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c1) p1 = MaxPooling2D((2, 2))(c1) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p1) c2 = AlphaDropout(0.1)(c2) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c2) p2 = MaxPooling2D((2, 2))(c2) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p2) c3 = AlphaDropout(0.2)(c3) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c3) p3 = MaxPooling2D((2, 2))(c3) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p3) c4 = AlphaDropout(0.2)(c4) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c4) p4 = MaxPooling2D(pool_size=(2, 2))(c4) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p4) c5 = AlphaDropout(0.3)(c5) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c5) u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5) u6 = concatenate([u6, c4]) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u6) c6 = AlphaDropout(0.2)(c6) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c6) u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6) u7 = concatenate([u7, c3]) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u7) c7 = AlphaDropout(0.2)(c7) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c7) u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7) u8 = concatenate([u8, c2]) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u8) c8 = AlphaDropout(0.1)(c8) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c8) u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8) u9 = concatenate([u9, c1], axis=3) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u9) c9 = AlphaDropout(0.1)(c9) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c9) outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9) model = Model(inputs=[inputs], outputs=[outputs]) return model
#classifier.add(AlphaDropout(0.1)) # Adding a fourth convolutional layer classifier.add( Conv2D(256, (3, 3), kernel_initializer='lecun_normal', padding='same')) classifier.add(Activation('selu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) #classifier.add(AlphaDropout(0.05)) # Step 3 - Flattening classifier.add(Flatten()) # Step 4 - Full connection classifier.add(Dense(units=128, kernel_initializer='lecun_normal')) classifier.add(Activation('selu')) classifier.add(AlphaDropout(0.2)) classifier.add(Dense(units=256, kernel_initializer='lecun_normal')) classifier.add(Activation('selu')) classifier.add(AlphaDropout(0.2)) #output layer classifier.add(Dense(units=1, activation='sigmoid')) # Compiling the CNN classifier.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy']) # Part 2 - Fitting the CNN to the images
def main( data=None, params={ 'lr': 0.001, 'conv_dr': 0., 'fc_dr': 0.1, 'batch_size': 128, 'no_epochs': 1000, 'steps_per_epoch': 100, 'dp_prob': 0.5, 'batch_norm': False, 'regularise': 0.0, 'decay': 0.0 }, no_threads=10, verbose=True, cp_interval=100, test=False): """ Runs a self-normalising neural network on the waveform data, saving the model to the filestore Requires a directory called 'logs' in the same folder for the TensorBoard visualisation The current neural network structure is: fc > dropout > fc > dropout > fc > dropout > fc > dropout > softmax Arguments: data - the Datasets object to run on, if None then loads data (default = None) params - a dictionary object containing the following parameters lr - the learning rate of the Adam optimiser (default = 0.001) conv_dr - unused fc_dr - the dropout rate for the alpha dropout layers (default = 0.1) no_epochs - the number of epochs to run for steps_per_epoch - the number of batches in each epoch dp_prob - the proportion of double pulse waveforms shown at train time (default = 0.5) batch_norm - unused regularise - sets the amount of L2 regularisation for each layer (default = 0.0) decay - sets the decay rate for the proportion of double-pulse waveforms used for training and validation (default = 0.0) no_threads - number of threads to use (default is 10, use -1 to set no limit) verbose - dictates the amount of output that keras gives cp_interval - the number of epochs between saving model checkpoints (default = 100) test - suppresses saving of model and output of logs (for testing new features; default = False) No returns """ # Read in data if data == None: data = load_data(verbose=verbose) # Set up CPU, GPU options config = None if no_threads == -1: config = tf.ConfigProto(allow_soft_placement=True, device_count={ 'CPU': 1, 'GPU': 1 }, gpu_options=tf.GPUOptions(allow_growth=True)) else: config = tf.ConfigProto(intra_op_parallelism_threads=no_threads, inter_op_parallelism_threads=no_threads, allow_soft_placement=True, device_count={ 'CPU': 1, 'GPU': 1 }, gpu_options=tf.GPUOptions(allow_growth=True)) sess = tf.Session(config=config) K.set_session(sess) # Define model model = Sequential() # Set up regulariser regulariser = l2(params['regularise']) model.add( Dense(1024, input_shape=(128, ), activation='selu', kernel_regularizer=regulariser)) model.add(AlphaDropout(params['fc_dr'])) model.add(Dense(1024, activation='selu', kernel_regularizer=regulariser)) model.add(AlphaDropout(params['fc_dr'])) model.add(Dense(1024, activation='selu', kernel_regularizer=regulariser)) model.add(AlphaDropout(params['fc_dr'])) model.add(Dense(64, activation='selu', kernel_regularizer=regulariser)) model.add(AlphaDropout(params['fc_dr'])) model.add(Dense(2, activation='softmax')) # Set-up optimiser optimiser = Adam(lr=params['lr']) # Create model model.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['accuracy', precision, recall, f1, class_balance]) # Create generators for training, validation train_gen = WaveformGenerator(data.train, batch_size=params['batch_size'], balanced=True, dp_prob=params['dp_prob'], decay=params['decay']) val_gen = WaveformGenerator(data.val, batch_size=params['batch_size'], balanced=True, dp_prob=params['dp_prob'], decay=params['decay']) # Prepare callbacks callbacks = [train_gen, val_gen] if test == False: tb = TensorBoard(log_dir='logs', histogram_freq=0, write_graph=True) model_saver = ModelSaver(model, 'snn', params, verbose=verbose, period=cp_interval) callbacks += [tb, model_saver] # Train model model.fit_generator(train_gen, steps_per_epoch=params['steps_per_epoch'], epochs=params['no_epochs'], verbose=int(verbose), validation_data=val_gen, validation_steps=params['steps_per_epoch'], callbacks=callbacks) # Evaluate model test_preds = model.predict(data.val.waveforms, verbose=int(verbose)) print() print_metric_results(data.val.labels, test_preds, data.val.weights, data.val.ids, th=0.5) print_metric_results(data.val.labels, test_preds, data.val.weights, data.val.ids, th=0.9)
def getModel(): inputdata = Input((1, 64, 64, 64)) conv1 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(inputdata) conv1 = PReLU()(conv1) conv2 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(conv1) conv2 = PReLU()(conv2) conv3 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(conv2) conv3 = PReLU()(conv3) pool1 = MaxPooling3D(pool_size=(2, 2, 2), data_format='channels_first')(conv3) encode1 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(pool1) encode1 = PReLU()(encode1) encode2 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(encode1) encode2 = PReLU()(encode2) encode3 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(encode2) encode3 = PReLU()(encode3) encode4 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(encode3) encode4 = PReLU()(encode4) upsampling1 = UpSampling3D(size=(2, 2, 2), data_format='channels_first')(encode4) finalShape = upsampling1.shape originalShape = conv3.shape cropShape = int(originalShape[2]/2-finalShape[2]/2),int(originalShape[3]/2-\ finalShape[3]/2),int(originalShape[4]/2-finalShape[4]/2) crop1 = Cropping3D(cropping=cropShape, data_format='channels_first')(conv3) concatenate1 = concatenate([upsampling1, crop1], axis=1) dropout1 = AlphaDropout(0.1)(concatenate1) expand1 = Convolution3D(256, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(dropout1) expand1 = PReLU()(expand1) expand2 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(expand1) expand2 = PReLU()(expand2) expand3 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(expand2) expand3 = PReLU()(expand3) expand4 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(expand3) expand4 = PReLU()(expand4) expand5 = Convolution3D(128, kernel_size=3, data_format='channels_first', activation='selu', padding='valid', use_bias=True, kernel_initializer='lecun_normal')(expand4) expand5 = PReLU()(expand5) outputdata = Convolution3D(1, kernel_size=1, data_format='channels_first', activation='sigmoid', padding='valid', use_bias=True, kernel_initializer='glorot_normal')(expand5) model = Model(inputs=inputdata, outputs=outputdata) model.compile(optimizer=Adam(lr=LR), loss=diceCoef) print model.summary() return model
# Hyperparameters EPOCHS = 6 BATCH_SIZE = 32 LEARNING_RATE = 1.0e-3 # Data train_generator = generator(train_samples, batch_size=BATCH_SIZE) valid_generator = generator(valid_samples, batch_size=BATCH_SIZE) # Define the model #model = load_model("./model.h5") model = Sequential() model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=SHAPE)) model.add( Conv2D(2, (3, 3), activation='selu', kernel_initializer='lecun_normal')) model.add(MaxPooling2D((4, 4))) model.add(AlphaDropout(0.25)) model.add(Flatten()) model.add(Dense(1)) model.compile(loss='mse', optimizer=optimizers.Adam(lr=LEARNING_RATE)) # Train the model model.fit_generator(train_generator, steps_per_epoch=np.ceil(len(train_samples) / BATCH_SIZE), validation_data=valid_generator, validation_steps=np.ceil(len(train_samples) / BATCH_SIZE), epochs=EPOCHS) # Save the model model.save("./model.h5")
def resnet(X_train, X_test): Y_train = to_onehot( map(lambda x: mods.index(lbl[x][0]), range(X_train.shape[0]))) Y_test = to_onehot( map(lambda x: mods.index(lbl[x][0]), range(X_test.shape[0]))) print('training started') in_shp = list(X_train.shape[1:]) print(X_train.shape, in_shp) classes = mods # Resnet Architecture def residual_stack(x): def residual_unit(y, _strides=1): shortcut_unit = y # 1x1 conv linear y = layers.Conv1D(32, kernel_size=5, data_format='channels_first', strides=_strides, padding='same', activation='relu')(y) y = layers.BatchNormalization()(y) y = layers.Conv1D(32, kernel_size=5, data_format='channels_first', strides=_strides, padding='same', activation='linear')(y) y = layers.BatchNormalization()(y) # add batch normalization y = layers.add([shortcut_unit, y]) return y x = layers.Conv1D(32, data_format='channels_first', kernel_size=1, padding='same', activation='linear')(x) x = layers.BatchNormalization()(x) x = residual_unit(x) x = residual_unit(x) # maxpool for down sampling x = layers.MaxPooling1D(data_format='channels_first')(x) return x inputs = layers.Input(shape=in_shp) x = residual_stack(inputs) x = residual_stack(x) x = residual_stack( x) # Comment this when the input dimensions are 1/32 or lower x = residual_stack( x) # Comment this when the input dimensions are 1/16 or lower x = residual_stack( x) # Comment this when the input dimensions are 1/8 or lower x = Flatten()(x) x = Dense(128, kernel_initializer="he_normal", activation="selu", name="dense1")(x) x = AlphaDropout(0.1)(x) x = Dense(128, kernel_initializer="he_normal", activation="selu", name="dense2")(x) x = AlphaDropout(0.1)(x) x = Dense(len(classes), kernel_initializer="he_normal", activation="softmax", name="dense3")(x) x_out = Reshape([len(classes)])(x) model = models.Model(inputs=[inputs], output=[x_out]) model.compile(loss='categorical_crossentropy', optimizer='adam') model.summary() # Set up some params nb_epoch = 500 # number of epochs to train on batch_size = 1024 # training batch size # Train the Model filepath = 'models/weights_resnet.wts.h5' model = multi_gpu_model(model, gpus=3) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(lr=0.001), metrics=['accuracy']) history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=2, validation_split=0.25, callbacks=[ keras.callbacks.ModelCheckpoint( filepath, monitor='val_loss', verbose=0, save_best_only=True, mode='auto'), keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='auto') ]) # we re-load the best weights once training is finished model.load_weights(filepath) score = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=0) return score[1]
def train_and_test(X_train_word, X_train_syntax, Y_train, X_test_word, X_test_syntax, Y_test, nb_epochs, batch_size, learning_rate, no_of_labels, embedding_dims_word, embedding_dims_syntax, char_to_idx, filter_tuple, filter_tuple1, dropout, activation_value, length, label_dic, model_type, logdir, weights=None, weights_pos=None): # activation_value must be relu or tanh bm_scores = True binary = False main_input = Input(shape=(X_train_word.shape[1], ), dtype='int32', name='main_input') x = Embedding(len(char_to_idx), embedding_dims_word, weights=weights, input_length=X_train_word.shape[1])(main_input) if dropout > 0: x = Dropout(dropout)(x) list_cnn_nodes = get_conv_layer(x, filter_tuple, activation_value) if len(list_cnn_nodes) > 1: list_cnn_nodes = layers.Concatenate()(list_cnn_nodes) else: list_cnn_nodes = list_cnn_nodes[0] if dropout > 0: list_cnn_nodes = AlphaDropout(dropout)(list_cnn_nodes) # the start of syntax ---------------- main_input_syntax = Input(shape=(X_train_syntax.shape[1], X_train_syntax.shape[2]), dtype='int32', name='main_input_syntax') x_syntax = Syntax_Embedding(input_dim=label_dic, syntax_tree_label_dim=embedding_dims_syntax, depth_of_syntax_tree=150, input_length=length)(main_input_syntax) if dropout > 0: x_syntax = Dropout(dropout)(x_syntax) list_cnn_nodes_syntax = get_conv_layer_syntax(x_syntax, filter_tuple1, activation_value) if len(list_cnn_nodes_syntax) > 1: list_cnn_nodes_syntax = layers.Concatenate()(list_cnn_nodes_syntax) else: list_cnn_nodes_syntax = list_cnn_nodes_syntax[0] if dropout > 0: list_cnn_nodes_syntax = AlphaDropout(dropout)(list_cnn_nodes_syntax) # the end of syntax ----------------- ###how to concat list_final = [] list_final.append(list_cnn_nodes) list_final.append(list_cnn_nodes_syntax) list_cnn_nodes = layers.Concatenate()(list_final) main_loss = Dense(no_of_labels, activation='softmax', name='main_output')(list_cnn_nodes) model = Model(inputs=[main_input, main_input_syntax], output=main_loss) print("model") timestamp = time.strftime("%Y%m%d-%H%M%S") # checkpointer = ModelCheckpoint(save_best_only=True, monitor='val_acc', verbose=1, filepath=os.path.join(logdir, model_type+timestamp+".hdf5"), ) # dict_callbacks = {'checkpointer':checkpointer} # , callbacks=dict_callbacks.values() adam = Adam(lr=learning_rate) model.summary() if binary: model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy']) else: model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) history = model.fit([X_train_word, X_train_syntax], Y_train, batch_size=batch_size, nb_epoch=300, validation_data=([X_test_word, X_test_syntax], Y_test)) open(os.path.join(logdir, model_type + timestamp + ".json"), 'w').write(model.to_json()) model.summary()
Y_TRAIN = to_categorical(Y_TRAIN, N_CLASSES) Y_TEST = to_categorical(Y_TEST, N_CLASSES) print('X_TRAIN shape --> {0}'.format(X_TRAIN.shape)) print('Y_TRAIN shape --> {0}'.format(Y_TRAIN.shape)) print('X_TEST shape --> {0}'.format(X_TEST.shape)) print('Y_TEST shape --> {0}'.format(Y_TEST.shape)) IMG_SHAPE = X_TRAIN[0].shape MODEL = Sequential() MODEL.add(InputLayer(input_shape=IMG_SHAPE)) MODEL.add(Dense(512)) MODEL.add(Activation('selu')) MODEL.add(AlphaDropout(0.2)) MODEL.add(Dense(512)) MODEL.add(Activation('selu')) MODEL.add(AlphaDropout(0.2)) MODEL.add(Dense(N_CLASSES)) MODEL.add(Activation('softmax')) MODEL.summary() plot_model(MODEL, to_file='../bin/snn_mnist_2_hidden.png') with open('../bin/snn_mnist_2_hidden.yaml', 'w') as f: f.write(MODEL.to_yaml()) MODEL.compile(loss=categorical_crossentropy, optimizer=RMSprop(),