def get_batchnorm_model(conv_model, p): model = Sequential([ MaxPooling2D(input_shape=conv_model.layers[-1].output_shape[1:]), Flatten(), Dense(4096, activation='relu'), BatchNormalization(), Dropout(p), Dense(4096, activation='relu'), BatchNormalization(), Dropout(p), Dense(1000, activation='softmax') ]) load_dense_weights_from_vgg16bn(model) for layer in model.layers: if type(layer) == Dense: layer.set_weights(process_weights(layer, 0.5, 0.6)) model.pop() for layer in model.layers: set_trainable(layer, False) model.add(Dense(10, activation='softmax')) return model
def _lstm(categorical=False, vocab_size=False, seq_len=200, embedding_length=10, nodes=100, lstm_drop=0.2, dropout=0.5): model = Sequential() model.add( Embedding(input_dim=vocab_size, output_dim=embedding_length, input_length=seq_len)) model.add(LSTM(nodes, dropout=dropout, recurrent_dropout=lstm_drop)) # if user requests no embedding, replace w/ LSTM only if not embedding_length: model.pop() model.pop() model.add( LSTM(nodes, dropout=dropout, recurrent_dropout=lstm_drop, input_shape=(seq_len, vocab_size))) if not categorical: model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) else: model.add(Dense(categorical, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def test_sequential_fit_generator(): (x_train, y_train), (x_test, y_test) = _get_test_data() def data_generator(train): if train: max_batch_index = len(x_train) // batch_size else: max_batch_index = len(x_test) // batch_size i = 0 while 1: if train: yield (x_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size]) else: yield (x_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size]) i += 1 i = i % max_batch_index model = Sequential() model.add(Dense(num_hidden, input_shape=(input_dim,))) model.add(Activation('relu')) model.add(Dense(num_classes)) model.pop() model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit_generator(data_generator(True), 5, epochs) model.fit_generator(data_generator(True), 5, epochs, validation_data=(x_test, y_test)) model.fit_generator(data_generator(True), 5, epochs, validation_data=data_generator(False), validation_steps=3) model.fit_generator(data_generator(True), 5, epochs, max_queue_size=2) model.evaluate(x_train, y_train)
def train_stacked_autoencoder(dataset): # Parâmetros das camadas input_dim = dataset.shape[1] hidden_dim = 10 activation_function = 'sigmoid' train_epochs = 100 # TODO # Modelo sae = Sequential() sae.add( Dense(hidden_dim, activation=activation_function, input_shape=(input_dim, ))) sae.add(Dense(input_dim, activation=activation_function)) sae.compile(optimizer='adam', loss='mse') sae.fit(dataset, dataset, epochs=train_epochs, verbose=0) for i in range(3): sae.pop() previous_layer = sae.get_layer(index=-1) previous_layer.trainable = False sae.add(Dense(hidden_dim, activation=activation_function)) sae.add(Dense(input_dim, activation=activation_function)) sae.compile(optimizer='adam', loss='mse') sae.fit(dataset, dataset, epochs=train_epochs, verbose=0) return sae
def test_sequential_fit_generator(): (x_train, y_train), (x_test, y_test) = _get_test_data() def data_generator(train): if train: max_batch_index = len(x_train) // batch_size else: max_batch_index = len(x_test) // batch_size i = 0 while 1: if train: yield (x_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size]) else: yield (x_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size]) i += 1 i = i % max_batch_index model = Sequential() model.add(Dense(num_hidden, input_shape=(input_dim,))) model.add(Activation('relu')) model.add(Dense(num_class)) model.pop() model.add(Dense(num_class)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit_generator(data_generator(True), 5, epochs) model.fit_generator(data_generator(True), 5, epochs, validation_data=(x_test, y_test)) model.fit_generator(data_generator(True), 5, epochs, validation_data=data_generator(False), validation_steps=3) model.fit_generator(data_generator(True), 5, epochs, max_queue_size=2) model.evaluate(x_train, y_train)
def build_model(): model = Sequential() model.add(Embedding(input_dim=vocab_size, output_dim=emb_dim, input_length=maxlen, weights=[embedding_weights])) model.add(Dropout(0.2)) model.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(300,))) model.add(Dropout(0.5)) model.add(Dense(512)) model.add(BatchNormalization()) model.add(PReLU()) model.add(Dropout(0.5)) model.add(RepeatVector(maxlen)) model.add(TimeDistributed(Dense(300))) model.add(BatchNormalization()) model.add(PReLU()) model.add(Dropout(0.5)) model.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) model.load_weights(weights_path) for _ in range(6): model.pop() model.add(Dense(nb_labels, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def build_model(sae_hidden_layers, INPUT_DIM, SAE_ACTIVATION, SAE_BIAS, SAE_OPTIMIZER, SAE_LOSS, batch_size, epochs, VERBOSE, x_train, y_train, path_sae_model): print("\nPart 1: buidling an SAE encoder ...") if False: # if os.path.isfile(path_sae_model) and (os.path.getmtime(path_sae_model) > os.path.getmtime(__file__)): model = load_model(path_sae_model) else: # create a model based on stacked autoencoder (SAE) model = Sequential() model.add( Dense(sae_hidden_layers[0], input_dim=INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) for units in sae_hidden_layers[1:]: model.add( Dense(units, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) model.add( Dense(INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) model.compile(optimizer=SAE_OPTIMIZER, loss=SAE_LOSS) # train the model model.fit(x_train, x_train, batch_size=batch_size, epochs=epochs, verbose=VERBOSE) # remove the decoder part num_to_remove = (len(sae_hidden_layers) + 1) // 2 for i in range(num_to_remove): model.pop() return model
def VGG16(ROOTPATH): # Returns # Determine proper input shape if K.image_dim_ordering() == 'th': INPUT_SHAPE = (3, 224, 224) else: INPUT_SHAPE = (224, 224, 3) model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=INPUT_SHAPE)) model.add(Convolution2D(64, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, 3, 3, activation='relu', trainable=False)) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu', trainable=False)) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', trainable=False)) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', trainable=False)) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(4096, activation='relu', trainable=True)) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu', trainable=True)) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax', trainable=True)) weights_path = ROOTPATH + "vgg16_weights_init.h5" print "LOAD: " + weights_path model.load_weights(weights_path) model.pop() # remove softmax layer model.pop() # remove dropout return model
def VGG_19(weights_path=None,onlyconv=False): model = Sequential() model.add(ZeroPadding2D((1,1),batch_input_shape=(1,224,224,3))) #0 model.add(Conv2D(64, (3, 3), activation='relu')) #1 model.add(ZeroPadding2D((1,1))) #2 model.add(Conv2D(64, (3, 3), activation='relu')) #3 model.add(MaxPooling2D((2,2), strides=(2,2))) #4 model.add(ZeroPadding2D((1,1))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3,3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3,3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) if weights_path: #print model.get_weights()[0][0][0] model.load_weights(weights_path) #print model.get_weights()[0][0][0] if onlyconv: for i in range(7): # get rid of fc layers model.pop() return model
def remove_layer(): model = Sequential() model.add(Dense(32, activation='relu', input_dim=784)) model.add(Dense(32, activation='relu')) print(len(model.layers)) model.pop() print(len(model.layers))
def buildModel(self, modelPath): layers = [] layers.append(ZeroPadding2D((1,1), input_shape=(3, 224, 224), name="input")) layers.append(Conv2D(64, kernel_size=(3 , 3), activation='relu', name='conv1_1')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(64, kernel_size=(3 , 3), activation='relu', name='conv1_2')) layers.append(MaxPooling2D((2, 2), strides=(2, 2))) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(128, kernel_size=(3 , 3), activation='relu', name='conv2_1')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(128, kernel_size=(3 , 3), activation='relu', name='conv2_2')) layers.append(MaxPooling2D((2, 2), strides=(2, 2))) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(256, kernel_size=(3 , 3), activation='relu', name='conv3_1')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(256, kernel_size=(3 , 3), activation='relu', name='conv3_2')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(256, kernel_size=(3 , 3), activation='relu', name='conv3_3')) layers.append(MaxPooling2D((2, 2), strides=(2, 2))) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv4_1')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv4_2')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv4_3')) layers.append(MaxPooling2D((2, 2), strides=(2, 2))) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv5_1')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv5_2')) layers.append(ZeroPadding2D((1, 1))) layers.append(Conv2D(512, kernel_size=(3 , 3), activation='relu', name='conv5_3')) layers.append(MaxPooling2D((2, 2), strides=(2, 2))) layers.append(Flatten(name='flat')) layers.append(Dense(4096, activation='relu', name='fc6')) layers.append(Dropout(0.5, name='dopout0')) layers.append(Dense(4096, activation='relu', name='fc7')) layers.append(Dropout(0.5, name='dropout1')) layers.append(Dense(1000, activation='softmax', name='softmax')) model = Sequential() for layer in layers: model.add(layer) while model.layers[-1].name != self.layerKey: model.pop() model.load_weights(modelPath, by_name=True) self.model = TimeDistributed(model)
def VGG_16(spatial_size, classes, channels, channel_first=True, weights_path=None): model = Sequential() if channel_first: model.add(ZeroPadding2D((1,1),input_shape=(channels, spatial_size, spatial_size))) else: model.add(ZeroPadding2D((1,1),input_shape=(spatial_size, spatial_size, channels))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(256, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Conv2D(512, (3, 3), activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) # 33 model.add(Flatten()) model.add(Dense(4096, activation='relu')) # 34 model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) # 35 model.add(Dropout(0.5)) model.add(Dense(2622, activation='softmax')) # Dropped if weights_path: model.load_weights(weights_path) model.pop() model.add(Dense(classes, activation='softmax')) # 36 return model
def remove_Dense_from_model(): model = Sequential() model.add(Dense(32, activation='relu', input_dim=784)) model.add(Dense(32, activation='relu')) print(model.layers) # "2" model.pop() print(model.layers) # "1"
def VGG_19_1(weights_path=None,onlyconv=False,caffe=False): model = Sequential() model.add(Conv2D(64, (3, 3), name='conv1_1', padding='same',activation='relu', batch_input_shape=(1,224,224,3), trainable=False)) #1 model.add(Conv2D(64, (3, 3), name='conv1_2', padding='same', activation='relu', trainable=False)) #3 model.add(AveragePooling2D((2,2), strides=(2,2))) #4 model.add(Conv2D(128, (3, 3), name='conv2_1', padding='same', activation='relu', trainable=False)) model.add(Conv2D(128, (3, 3), name='conv2_2', padding='same', activation='relu', trainable=False)) model.add(AveragePooling2D((2,2), strides=(2,2))) model.add(Conv2D(256, (3,3), name='conv3_1', padding='same', activation='relu', trainable=False)) model.add(Conv2D(256, (3,3), name='conv3_2', padding='same', activation='relu', trainable=False)) model.add(Conv2D(256, (3,3), name='conv3_3', padding='same', activation='relu', trainable=False)) model.add(Conv2D(256, (3,3), name='conv3_4', padding='same', activation='relu', trainable=False)) model.add(AveragePooling2D((2,2), strides=(2,2))) model.add(Conv2D(512, (3,3), name='conv4_1', padding='same',activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv4_2', padding='same', activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv4_3', padding='same', activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv4_4', padding='same', activation='relu', trainable=False)) model.add(AveragePooling2D((2,2), strides=(2,2))) model.add(Conv2D(512, (3,3), name='conv5_1', padding='same', activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv5_2', padding='same', activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv5_3', padding='same', activation='relu', trainable=False)) model.add(Conv2D(512, (3,3), name='conv5_4', padding='same', activation='relu', trainable=False)) model.add(AveragePooling2D((2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) #model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) #model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) if caffe: weights_data = np.load("dataout.h5").item() for layer in model.layers: if layer.name in weights_data.keys(): #print 'loading layer',layer.name layer_weights = weights_data[layer.name] layer.set_weights((layer_weights['weights'], layer_weights['biases'])) else: if weights_path: #print model.get_weights()[0][0][0] model.load_weights(weights_path) #print model.get_weights()[0][0][0] if onlyconv: for i in range(5): # get rid of fc layers #print 'popping', model.layers[-1] model.pop() return model
def build_model(sae_hidden_layers, INPUT_DIM, SAE_ACTIVATION, SAE_BIAS, SAE_OPTIMIZER, SAE_LOSS, batch_size, epochs, VERBOSE, RSS_train, y_train): # create a model based on stacked autoencoder (SAE) model = Sequential() model.add( Dense(sae_hidden_layers[0], input_dim=INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) for units in sae_hidden_layers[1:]: model.add( Dense( units, activation=SAE_ACTIVATION, use_bias=SAE_BIAS, activity_regularizer=regilization, )) model.add(Dense( INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS, )) model.compile(optimizer=SAE_OPTIMIZER, loss=SAE_LOSS) # train the model model.fit(RSS_train, RSS_train, batch_size=batch_size, epochs=epochs, verbose=VERBOSE) # remove the decoder part num_to_remove = (len(sae_hidden_layers) + 1) // 2 for i in range(num_to_remove): model.pop() model.add(Dropout(dropout)) return model #print(INPUT_DIM) #input=Input(batch_shape = (batch_size,INPUT_DIM)) #x=Dense(sae_hidden_layers[0], input_dim=INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)(input) #for units in sae_hidden_layers[1:]: # x=Dense(units, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)(x) #x=Dense(INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)(x) #model = Model(inputs = input, outputs = x) #model.compile(optimizer=SAE_OPTIMIZER, loss=SAE_LOSS) #model.fit(RSS_train, RSS_train, batch_size=batch_size, epochs=epochs, verbose=VERBOSE) ## # remove the decoder part #num_to_remove = (len(sae_hidden_layers) + 1) // 2 #for i in range(num_to_remove): # model.pop() return model
def ready_model(weights_path, n_classes): model = Sequential() model.add(ZeroPadding2D((1,1),input_shape=(3,224,224))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1,1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(365, activation='softmax')) model.load_weights(weights_path) # Freeze all layers except the last two model.pop() model.pop() model.add(Dropout(0.5)) model.add(Dense(n_classes, activation='softmax')) sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy',metrics=['accuracy']) return model
def test_GRU_layer(self): model = Sequential() model.add(Embedding(1000, 64, input_length=10)) model.add(GRU(128)) self.assertEqual(model.get_layer(index=1).units, 128) self.assertEqual(model.get_layer(index=1).input_shape, (None, 10, 64)) self.assertEqual(model.get_layer(index=1).output_shape, (None, 128)) model.pop() model.add(GRU(128, return_sequences=True)) self.assertEqual( model.get_layer(index=1).output_shape, (None, 10, 128))
def test_Bidirectional_wrapper(self): model = Sequential() model.add(Embedding(1000, 64, input_length=10)) model.add(Bidirectional(LSTM(128))) self.assertEqual(model.get_layer(index=1).forward_layer.units, 128) self.assertEqual(model.get_layer(index=1).backward_layer.units, 128) self.assertEqual(model.get_layer(index=1).input_shape, (None, 10, 64)) self.assertEqual(model.get_layer(index=1).output_shape, (None, 256)) model.pop() model.add(Bidirectional(LSTM(128, return_sequences=True))) self.assertEqual( model.get_layer(index=1).output_shape, (None, 10, 256))
def VGG_16(weights_path=None, save_path=None): model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=(3, 224, 224))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) if weights_path: model.load_weights(weights_path) model.pop() model.add(Dense(20, activation='softmax')) model.save(save_path, overwrite=True) return model
def trainDBN(layers,xTrain): pop = 1 push = 0 model = Sequential() opt = Adam(lr=0.01) for i in range(len(layers)-1): auxLayers = list() if push != 0: # Mark layers as non-trainable for layer in model.layers: layer.trainable = False for j in range(pop): # remember the current output layer auxLayers.append(model.layers[-1]) # remove the output layer for layer in model.layers: print(layer) model.pop() # Generate new hidden layers if push == 0: model.add(Dense(input_shape=(773,), units=layers[i], activation='relu')) model.add(Dense(units=layers[i+1], activation='relu')) model.add(Dense(units=layers[i], activation='relu')) else: model.add(Dense(units=layers[i], activation='relu')) model.add(Dense(units=layers[i+1], activation='relu')) model.add(Dense(units=layers[i], activation='relu')) # push if push != 0: # push saved layers for k in range(push): model.add(auxLayers[push-1-k]) # Compile the model model.compile(optimizer=opt, loss='mean_squared_error', metrics=['accuracy']) # Train NN num_epochs = 200 batch_size = 256 history = model.fit(x=xTrain, y=xTrain, epochs=num_epochs, batch_size=batch_size, shuffle=True, validation_data=(xTrain, xTrain), verbose=1) pop += 1 push += 1 model.summary() # Fix possible problems with new model model.save('temp.h5') model = load_model('temp.h5') return model
def modelTrain(originalFeature, target, name, predValue, saveT=True, ifTrain=True, batch_size=512, epochs=20): model = Sequential() model.add(Dense(50, input_shape=(originalFeature.shape[1], ))) model.add(Dense(100, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(500, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(10, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(1, activation="relu")) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # model.summary() if ifTrain: X_tra, X_val, y_tra, y_val = train_test_split(originalFeature, target, test_size=0.4, random_state=42) hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=1) print('-' * 400) name = str(name) + '.h5' if saveT: model.save_weights(name) else: model.load_weights(name) if predValue.shape[1] != originalFeature.shape[1]: print('check !!!!') print(predValue.shape) print(originalFeature.shape) model.pop() model.pop() # model.summary() output = model.predict(predValue, batch_size=batch_size, verbose=1) return output
def test_sequential_pop(): model = Sequential() model.add(Dense(num_hidden, input_dim=input_dim)) model.add(Dense(num_classes)) model.compile(loss='mse', optimizer='sgd') x = np.random.random((batch_size, input_dim)) y = np.random.random((batch_size, num_classes)) model.fit(x, y, epochs=1) model.pop() assert len(model.layers) == 1 assert model.output_shape == (None, num_hidden) model.compile(loss='mse', optimizer='sgd') y = np.random.random((batch_size, num_hidden)) model.fit(x, y, epochs=1)
def train_model(x_train, y_train, savepath, load_model=False, save_model=False, epochs=20, verbose=0): # Create neural net model = Sequential() model.add( Dense(64, input_dim=x_train.shape[1], kernel_initializer='normal')) model.add(BatchNormalization()) model.add(Activation("relu")) model.add(Dense(32, kernel_initializer='normal')) model.add(BatchNormalization()) model.add(Activation("relu")) model.add(Dense(16, kernel_initializer='normal')) model.add(BatchNormalization()) model.add(Activation("relu")) model.add(Dense(8, kernel_initializer='normal')) model.add(BatchNormalization()) model.add(Activation("relu")) model.add(Dense(y_train.shape[1], activation='softmax')) if (load_model): model.load_weights(savepath) model.pop() model.add(Dense(y_train.shape[1], activation='softmax')) #pout(model.summary()) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) #monitor = EarlyStopping(monitor='loss', min_delta=1e-3, patience=5, verbose=1, mode='auto') #checkpointer = callbacks.ModelCheckpoint(filepath="Checkpoints/checkpoint-{epoch:02d}.hdf5", verbose=0, save_best_only=True, monitor='loss') #csv_logger = CSVLogger('Checkpoints/training_set_dnnanalysis.csv',separator=',', append=False) model.fit(x_train, y_train, verbose=verbose, epochs=epochs) if (save_model == True): model.save_weights(savepath) return model
def _cnn(categorical=False, vocab_size=False, seq_len=150, embedding_length=10, cnn_filters=128, filter_length=3, pool_size=2, nodes=100, dropout=0.5): model = Sequential() model.add( Embedding(input_dim=vocab_size, output_dim=embedding_length, input_length=seq_len)) model.add( Conv1D(filters=cnn_filters, kernel_size=filter_length, padding='same', activation='selu')) # if user requests no embedding, replace w/ CNN only if not embedding_length: model.pop() model.pop() model.add( Conv1D(filters=cnn_filters, kernel_size=filter_length, input_shape=(seq_len, vocab_size), padding='same', activation='selu')) model.add(MaxPooling1D(pool_size=pool_size)) model.add(Flatten()) model.add(Dense(nodes)) model.add(Dropout(dropout)) if not categorical: model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) else: model.add(Dense(categorical, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def get_fc_model_batchnorm(p, conv_layers): bn_model = Sequential(get_bn_layers(p, conv_layers)) load_fc_weights_from_vgg16bn(bn_model) for l in bn_model.layers: if type(l) == Dense: l.set_weights(proc_bn_wgts(l, 0.5, 0.6)) # Remove last layer and lock all the others bn_model.pop() for layer in bn_model.layers: layer.trainable = False # Add linear layer (2-class) (just doing the ImageNet mapping to Kaggle dogs and cats) bn_model.add(Dense(2, activation='softmax')) bn_model.compile(Adam(), 'categorical_crossentropy', metrics=['accuracy']) # NOTE: Adam optimizer return bn_model
def vgg_feature_model(): def preprocess(img): vgg_mean = np.array([123.68, 116.779, 103.939]).reshape((3, 1, 1)) return (img - vgg_mean)[:, ::-1] # 注意第一个维度是batch_size def AddConvBlock(model, layers, filters): for _ in range(layers): model.add(ZeroPadding2D((1, 1))) model.add(Conv2D(filters, kernel_size=(3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) def AddFCBlock(model, units, dropout=0.5): model.add(Dense(units, activation='relu')) if dropout is not None: model.add(Dropout(dropout)) vgg_model = Sequential() # 预处理:这里要指定输入张量的维度。在后面的模块中一般不需要考虑上一层的输入维度,keras会自动计算 vgg_model.add( Lambda(preprocess, input_shape=(3, 224, 224), output_shape=(3, 224, 224))) # 添加卷积模块 AddConvBlock(vgg_model, 2, 64) AddConvBlock(vgg_model, 2, 128) AddConvBlock(vgg_model, 3, 256) AddConvBlock(vgg_model, 3, 512) AddConvBlock(vgg_model, 3, 512) # 将(channels, height, width)的三维张量打平成(channels * height * width, )的一维张量 vgg_model.add(Flatten()) # 添加全连接层和dropout AddFCBlock(vgg_model, units=4096, dropout=0.5) AddFCBlock(vgg_model, units=4096, dropout=0.5) # the last layer: softmax layer vgg_model.add(Dense(units=1000, activation="softmax")) vgg_model.load_weights("./models/vgg16.h5") # pop the fc layer and use flatten layer as output for feature extraction while type(vgg_model.layers[-1]) is not Flatten: vgg_model.pop() return vgg_model
def fcc_auto_model(maxlen, vocab_size, filepath, dense_outputs, classes): #Define what the input shape looks like model = Sequential() model.add(Dense(128, activation='relu', input_shape=(maxlen, vocab_size), trainable = False)) model.add(Dense(64, activation='relu', trainable = False)) model.add(Dropout(0.3, trainable = False)) model.add(Dense(32, activation='relu', trainable = False)) model.add(Dense(64, activation='relu', trainable = False)) model.add(Dense(128, activation='relu', trainable = False)) model.add(Dense(vocab_size, activation='sigmoid', trainable = False)) model.load_weights(filepath) model.pop() # Dense(vocab_size, activation='sigmoid')(decoded) model.pop() # Dense(128, activation='relu')(decoded) model.pop() # Dense(64, activation='relu')(encoded) model.add(Flatten()) model.add(Dense(32, activation='relu', name = 'dense1')) model.add(Dropout(0.5, name = 'dropout1')) model.add(Dense(classes, activation='softmax', name='output')) model.compile(optimizer='adadelta', loss='categorical_crossentropy',metrics=['accuracy']) return (model, filepath)
def test_sequential_fit_generator(): (X_train, y_train), (X_test, y_test) = _get_test_data() def data_generator(train): if train: max_batch_index = len(X_train) // batch_size else: max_batch_index = len(X_test) // batch_size i = 0 while 1: if train: yield (X_train[i * batch_size : (i + 1) * batch_size], y_train[i * batch_size : (i + 1) * batch_size]) else: yield (X_test[i * batch_size : (i + 1) * batch_size], y_test[i * batch_size : (i + 1) * batch_size]) i += 1 i = i % max_batch_index model = Sequential() model.add(Dense(nb_hidden, input_shape=(input_dim,))) model.add(Activation("relu")) model.add(Dense(nb_class)) model.pop() model.add(Dense(nb_class)) model.add(Activation("softmax")) model.compile(loss="categorical_crossentropy", optimizer="rmsprop") model.fit_generator(data_generator(True), len(X_train), nb_epoch) model.fit_generator(data_generator(True), len(X_train), nb_epoch, validation_data=(X_test, y_test)) model.fit_generator( data_generator(True), len(X_train), nb_epoch, validation_data=data_generator(False), nb_val_samples=batch_size * 3, ) model.fit_generator(data_generator(True), len(X_train), nb_epoch, max_q_size=2) loss = model.evaluate(X_train, y_train)
def sphere_neurons(model): for layer in range(len(model.layers)): if 'conv2d' in model.layers[layer].name: modelx = Sequential() modelx.add(HB(hbp / 2, shift = -1, input_shape = (img_rows, img_cols, 1), unif = unif, just_dropout = just_dropout)) modelx.add(Conv2D(nb_filters, kernel_size = (nb_conv, nb_conv), padding = 'valid', activation = 'relu')) modelx.add(HB(hbp, shift = -2, unif = unif, just_dropout = just_dropout)) modelx.add(Conv2D(nb_filters, kernel_size = (nb_conv, nb_conv), padding = 'valid', activation = 'relu')) modelx.add(MaxPooling2D((nb_pool, nb_pool))) modelx.add(HB(hbp, shift = -3, unif = unif, just_dropout = just_dropout)) modelx.add(Conv2D(nb_filters, kernel_size = (nb_conv, nb_conv), padding = 'valid', activation = 'relu')) modelx.add(HB(hbp, shift = -4, unif = unif, just_dropout = just_dropout)) modelx.add(Conv2D(nb_filters, kernel_size = (nb_conv, nb_conv), padding = 'valid', activation = 'relu')) modelx.add(MaxPooling2D((2, 2))) modelx.add(Flatten()) modelx.add(HB(hbp, shift = -6, unif = unif, just_dropout = just_dropout)) modelx.add(Dense(nb_classes, activation = 'softmax')) sgd = SGD(lr = 0.01, momentum = 0.9, decay = 0.00, nesterov = False) for i in range((len(model.layers) - 1) - layer): modelx.pop() modelx.compile(loss ='categorical_crossentropy', optimizer = sgd, metrics = ['accuracy']) modelx.set_weights(model.get_weights()) preds = modelx.predict(small_X_train) channels_output = [preds[:, :, :, channel].flatten() for channel in range(nb_filters)] channels_output = np.array(channels_output).T channels_cov = np.cov(channels_output, rowvar = False) sphere_channels = fractional_matrix_power(channels_cov, -1 / 2.) current_weights = model.layers[layer].get_weights() for i in range(nb_filters): new_weight = sphere_channels[i, 0] * modelx.layers[layer].get_weights()[0][:, :, :, 0] for j in range(1, nb_filters): new_weight += sphere_channels[i, j] * model.layers[layer].get_weights()[0][:, :, :, j] current_weights[0][:, :, :, i] = new_weight model.layers[layer].set_weights(current_weights) print(layer)
class VGG16(object): def __init__(self, drop=0., pretrained_weight_path=None): self._get_original_vgg16(drop) if pretrained_weight_path: self.fine_tune() self.model.load_weights(pretrained_weight_path) self.compile() print('model loaded from path={}'.format(pretrained_weight_path)) def _conv_block(self, nb_filter, nb_conv): for _ in range(nb_conv): self.model.add(Convolution2D(nb_filter, 3, 3, activation='relu', border_mode='same')) self.model.add(MaxPooling2D((2, 2))) def _fc_block(self, drop): self.model.add(Dense(4096, activation='relu')) self.model.add(Dropout(drop)) def _get_original_vgg16(self, drop): self.model = Sequential() self.model.add(Lambda(vgg_preprocess, input_shape=( 3, 224, 224), output_shape=(3, 224, 224))) self._conv_block(64, 2) self._conv_block(128, 2) self._conv_block(256, 3) self._conv_block(512, 3) self._conv_block(512, 3) self.model.add(Flatten()) self._fc_block(drop) self._fc_block(drop) self.model.add(Dense(1000, activation='relu')) weights = get_file('vgg16_weights', TH_WEIGHTS_PATH, cache_subdir='models') self.model.load_weights(weights) def fine_tune(self): self.model.pop() # take the final dense(1000) out and attach your output layer for layer in self.model.layers: layer.trainable = False self.model.add(Dense(2, activation='softmax')) def compile(self, lr=0.01): self.model.compile(optimizer=Adam(lr=lr), loss='categorical_crossentropy', metrics=['accuracy']) def get_batch_generator(self, path, batch_size=32, shuffle=True, class_mode='categorical', augmentation=False): if augmentation: gen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.1, horizontal_flip=True) else: gen = ImageDataGenerator() return gen.flow_from_directory( path, target_size=(224, 224), batch_size=batch_size, class_mode=class_mode, shuffle=shuffle) def fit(self, train, valid, nb_epoch=1): self.model.fit_generator( train, samples_per_epoch=train.nb_sample, nb_epoch=nb_epoch, validation_data=valid, nb_val_samples=valid.nb_sample, verbose=2) def save_weight(self, path): self.model.save_weights(path) def predict_prob_generator(self, path): """ return the probability of dog array([[ 1.0000e+00, 9.4158e-18], [ 1.0000e+00, 9.7884e-16]) """ gen = self.get_batch_generator(path, shuffle=False) prob = self.model.predict_generator( gen, val_samples=gen.nb_sample) return prob[:, 1] def predict_label_generator(self, prob): return np.around(prob)
Dense(units, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) model.add( Dense(INPUT_DIM, activation=SAE_ACTIVATION, use_bias=SAE_BIAS)) model.compile(optimizer=SAE_OPTIMIZER, loss=SAE_LOSS) # train the model model.fit(x_train, x_train, batch_size=batch_size, epochs=epochs, verbose=VERBOSE) # remove the decoder part num_to_remove = (len(sae_hidden_layers) + 1) // 2 for i in range(num_to_remove): model.pop() # # set all layers (i.e., SAE encoder) to non-trainable (weights will not be updated) # for layer in model.layers[:]: # layer.trainable = False # save the model for later use model.save(path_sae_model) ### build and train a complete model with the trained SAE encoder and a new classifier print("\nPart 2: buidling a complete model ...") # append a classifier to the model model.add(Dropout(dropout)) for units in classifier_hidden_layers: model.add(
x_train = x_train / 255.0 x_test = x_test / 255.0 y_train = to_categorical(y_train) y_test = to_categorical(y_test) num_classes = y_test.shape[1] # In[105]: model = load_model('model_cifar-10.h5', custom_objects={ 'accuracy': accuracy, 'f1_': f1_, 'precision_': precision_, 'recall_': recall_ }) model.pop() #remove the last layer - 'Dense' layer with 10 units for layer in model.layers: layer.trainable = False model.add(Dense(units=num_classes, activation='softmax')) # Compile model epochs = 25 lrate = 0.01 decay = lrate / epochs sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy', f1_, precision_, recall_]) print(model.summary()) # In[106]:
print("===== Read data done =====") train_len = X_train.shape[0] train_batch_num = int(train_len/batch_size)+1 print("train_batch_num:", train_batch_num) valid_len = X_valid.shape[0] valid_batch_num = int(valid_len/batch_size)+1 print("valid_batch_num:", valid_batch_num) order = np.arange(train_len) e = int(sys.argv[1])+1 if sys.argv[1]!='start' else 1 for epoch in range(e, epoch_num+1): # Cooling down temperature = max(1.0, temperature-0.2) model.pop() # pop activation model.pop() # pop lambda temperature model.add(Lambda(lambda x: x/temperature)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0004), metrics=['accuracy']) print("Epoch:", epoch) np.random.shuffle(order) X = X_train[order] Y = Y_train[order] train_loss = 0.0 train_acc = 0.0 for b in range(train_batch_num): if b%200==0: print(b) b_x = X[b*batch_size:b*batch_size+batch_size] # (b, 50, 69)