def baseline_model_conv_2(train_X, train_y, test_X, test_y, num_pixels=256, num_classes=10, learningRate=0.001, reg=l1(0.0001)): # encode targets train_y_enc = to_categorical(train_y) test_y_enc = to_categorical(test_y) # define model model = Sequential() model.add( Conv2D(64, kernel_size=(3, 3), activation='elu', input_shape=(16, 16, 1), activity_regularizer=reg)) model.add(Dropout(0.25)) #model.add(Conv2D(64, (3, 3), activation='elu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dense(num_classes, activation='sigmoid')) opt = optimizers.nadam(lr=learningRate) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=opt, metrics=['accuracy']) return model
def build(self): word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH + 2), dtype="int32") inputs = [word_inputs] word_outputs = self.build_word_cnn(word_inputs) if len(self.word_vectorizers) > 0: additional_word_inputs = [ kl.Input(shape=(None, input_dim), dtype="float32") for input_dim, dense_dim in self.word_vectorizers ] inputs.extend(additional_word_inputs) additional_word_embeddings = [ kl.Dense(dense_dim)(additional_word_inputs[i]) for i, (_, dense_dim) in enumerate(self.word_vectorizers) ] word_outputs = kl.Concatenate()([word_outputs] + additional_word_embeddings) outputs, lstm_outputs = self.build_basic_network(word_outputs) compile_args = { "optimizer": ko.nadam(lr=0.002, clipnorm=5.0), "loss": "categorical_crossentropy", "metrics": ["accuracy"] } self.model_ = Model(inputs, outputs) self.model_.compile(**compile_args) if self.verbose > 0: log.info(str(self.model_.summary())) return self
def evaluate_model_dense_2(train_X, train_y, test_X, test_y, num_pixels=256, num_classes=10, learningRate=0.001, reg=l1(0.001)): # encode targets train_y_enc = to_categorical(train_y) test_y_enc = to_categorical(test_y) #have to replace with our models # define model model = Sequential() model.add( Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='elu', activity_regularizer=reg)) model.add(Dense(128, kernel_initializer='normal', activation='relu')) model.add(Dense(64, kernel_initializer='normal', activation='relu')) model.add(Dense(32, kernel_initializer='normal', activation='relu')) model.add(Dense(16, kernel_initializer='normal', activation='relu')) model.add( Dense(num_classes, kernel_initializer='normal', activation='sigmoid')) opt = optimizers.nadam(lr=learningRate) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) return model
def create_model(input_size, output_size, n_layers, n_neurons, activation_function, learning_rate, dropout_rate, optimizer): model = models.Sequential() model.add( layers.Dense(n_neurons, input_shape=(input_size, ), name='new_androdet_dense_1')) for _ in range(n_layers): if dropout_rate != 0.0: model.add(layers.Dropout(dropout_rate, noise_shape=None, seed=None)) model.add(layers.Dense(n_neurons, activation=activation_function)) model.add(layers.Dense(output_size, activation="sigmoid")) #model.summary() if optimizer == 'rmsprop': opt = optimizers.rmsprop(lr=learning_rate) elif optimizer == 'adam': opt = optimizers.adam(lr=learning_rate) elif optimizer == 'sgd': opt = optimizers.sgd(lr=learning_rate) elif optimizer == 'adagrad': opt = optimizers.adagrad(lr=learning_rate) elif optimizer == 'adadelta': opt = optimizers.adadelta(lr=learning_rate) elif optimizer == 'adamax': opt = optimizers.adamax(lr=learning_rate) elif optimizer == 'nadam': opt = optimizers.nadam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["mean_squared_error"]) return model
def createArchitecture(parameters): optimizer = 0 if parameters["optimizer"] == 'rmsprop': optimizer = optimizers.rmsprop(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'adam': optimizer = optimizers.adam(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'nadam': optimizer = optimizers.nadam(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'sgd': optimizer = optimizers.sgd(lr=parameters["learning_rate"]) #else: # optimizer = parameters["optimizer"] if parameters["use_embedding_layer"]: input = Input(shape=(parameters["max_seq_len"], )) model = Embedding(input_dim=parameters["one_hot_vector_len"], output_dim=parameters["embedding_layer_output"], input_length=parameters["max_seq_len"])(input) if parameters["embedding_dropout"] > 0: model = Dropout(rate=parameters["embedding_dropout"])(model) else: input = Input(shape=(parameters["max_seq_len"], parameters["one_hot_vector_len"])) model = input if parameters["bi_lstm1_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm1_units"], return_sequences=True))(model) if parameters["bi_lstm2_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm2_units"], return_sequences=True))(model) if parameters["bi_lstm3_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm3_units"], return_sequences=True))(model) if parameters["use_crf_layer"]: crf = CRF(parameters["num_tags"], learn_mode="marginal") out = crf(model) # output model = Model(input, out) model.compile(optimizer=optimizer, loss=losses.crf_loss, metrics=[metrics.crf_accuracy, avg_proximity_metric()]) else: out = TimeDistributed( Dense(parameters["num_tags"], activation="softmax"))(model) model = Model(input, out) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy", avg_proximity_metric()]) model.summary() return model
def startLearning(Env, max_board_size=7, loadFileNumber=-1, gpuToUse=None, memoryAllocation=800000): # Set to use GPU explicitly if gpuToUse != None: environ["CUDA_VISIBLE_DEVICES"]=gpuToUse else: environ["CUDA_VISIBLE_DEVICES"]="0" env = Env nb_actions = env.action_space.n # Init size based on max_board_size if max_board_size not in [11, 7, 19]: raise EnvironmentError layer0Size = 4096 layer1Size = 4096 layer2Size = 4096 layer3Size = 0 layer4Size = 0 layer5Size = 0 # Next, we build a very simple model. model = Sequential() model.add(Flatten(input_shape=(1,) + env.observation_space.shape)) model.add(Dense(layer0Size)) model.add(LeakyReLU(alpha=0.003)) model.add(Dense(layer1Size)) model.add(LeakyReLU(alpha=0.003)) model.add(Dense(layer2Size)) model.add(LeakyReLU(alpha=0.003)) model.add(Dense(nb_actions)) model.add(Activation('linear')) #A little diagnosis of the model summary print(model.summary()) # Finally, we configure and compile our agent. memory = SequentialMemory(limit=memoryAllocation, window_length=1) policy = BoltzmannQPolicy() dqn = DQNAgent(model=model, batch_size=32, nb_actions=nb_actions, memory=memory, policy=policy, enable_dueling_network=True, gamma=.97) dqn.compile(nadam(lr=0.01), metrics=['mae']) # Here we load from a file an old agent save if specified. if loadFileNumber >= 0: loadFile = "Larger_Memeory_BOARDSIZE_" + str(max_board_size) + "_DQN_LAYERS_" + str(layer0Size) + "_" + str(layer1Size) + "_" + str(layer2Size) + "_" + str(layer3Size) + "_" + str(layer4Size) + "_" + str(layer5Size) + "_SAVENUMBER_" + str(loadFileNumber) + ".h5f" dqn.load_weights(loadFile) saveFileNumberCounter = 0 while True: dqn.fit(env, nb_steps=100010, visualize=False, verbose=1) saveFileNumberCounter+=1 saveFile = "Larger_Memeory_BOARDSIZE_" + str(max_board_size) + "_DQN_LAYERS_" + str(layer0Size) + "_" + str(layer1Size) + "_" + str(layer2Size) + "_" + str(layer3Size) + "_" + str(layer4Size) + "_" + str(layer5Size) + "_SAVENUMBER_" + str(loadFileNumber + saveFileNumberCounter) + ".h5f" dqn.save_weights(saveFile, overwrite=True)
def model7(): base_model = ResNet50(weights=None, include_top=False, input_shape=(300, 300, 3)) base_model.load_weights('../input/keras-pretrained-models/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5') base_model.trainable = False model = Sequential([ base_model, GlobalAveragePooling2D(), Dense(6, activation='softmax') ]) opt = optimizers.nadam(lr=0.0001) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy']) return model
def learn_effec_model(num_pixels = 256, num_classes = 10, learning_rate=0.001): model = Sequential() model.add(Conv2D(32, kernel_size=(3, 3), kernel_initializer='glorot_uniform', activation='relu', input_shape=(16,16,1))) model.add(Conv2D(64, (3, 3), kernel_initializer='glorot_uniform', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, kernel_initializer='glorot_uniform', activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, kernel_initializer='glorot_uniform', activation='sigmoid')) optimizer = optimizers.nadam(lr=learning_rate) model.compile(loss=keras.losses.categorical_crossentropy,optimizer=optimizer,metrics=['accuracy']) return model
def model5(): # v19-0.88 base_model = InceptionV3(weights=None, include_top=False, input_shape=(300, 300, 3)) base_model.load_weights('../input/keras-pretrained-models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5') base_model.trainable = False model = Sequential([ base_model, GlobalAveragePooling2D(), Dropout(0.02), Dense(1024, activation='relu'), Dense(2, activation='softmax') ]) opt = optimizers.nadam(lr=0.0001) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy']) return model
def build(self): word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH + 2), dtype="int32") inputs = [word_inputs] word_outputs = self.build_word_cnn(word_inputs) outputs, lstm_outputs = self.build_basic_network(word_outputs) compile_args = { "optimizer": ko.nadam(lr=0.002, clipnorm=5.0), "loss": "categorical_crossentropy", "metrics": ["accuracy"] } self.model_ = Model(inputs, outputs) self.model_.compile(**compile_args) if self.verbose > 0: print(self.model_.summary()) return self
def create_model(layers_and_filters, kernels, activation, input_shape, dropout_rate, optimizer, learning_rate, output_size=1): model = models.Sequential() i = 0 for filters in layers_and_filters: model.add( layers.Conv2D(filters, kernel_size=kernels[i], strides=kernels[i], activation=activation, input_shape=input_shape)) i += 1 if i < len(layers_and_filters): model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add(layers.BatchNormalization()) if dropout_rate != 0: model.add(layers.Dropout(dropout_rate)) model.add(layers.Flatten()) model.add(layers.Dense(output_size, activation='sigmoid')) if optimizer == 'rmsprop': opt = optimizers.rmsprop(lr=learning_rate) elif optimizer == 'adam': opt = optimizers.adam(lr=learning_rate) elif optimizer == 'sgd': opt = optimizers.sgd(lr=learning_rate) elif optimizer == 'adagrad': opt = optimizers.adagrad(lr=learning_rate) elif optimizer == 'adadelta': opt = optimizers.adadelta(lr=learning_rate) elif optimizer == 'adamax': opt = optimizers.adamax(lr=learning_rate) elif optimizer == 'nadam': opt = optimizers.nadam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["mean_squared_error"]) #model.summary() return model
def nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004): """ Adam optimizer :param lr: >=0, learning rate :param beta_1: 0 < beta_1 < 1, generally close to 1 :param beta_2: 0 < beta_2 < 1, generally close to 1 :param epsilon: >=0, fuzz factor. If None, defaults to K.epsilon() """ return optimizers.nadam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, schedule_decay=schedule_decay)
def learn_effec_model(num_pixels=256, num_classes=10, learning_rate=0.001): model = Sequential() model.add( Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='elu')) model.add(Dense(128, kernel_initializer='normal', activation='elu')) model.add(Dense(64, kernel_initializer='normal', activation='relu')) model.add(Dense(32, kernel_initializer='normal', activation='relu')) model.add(Dense(16, kernel_initializer='normal', activation='relu')) model.add( Dense(num_classes, kernel_initializer='normal', activation='sigmoid')) optimizer = optimizers.nadam(lr=learning_rate) #optimizer = optimizers.SGD(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) return model
def __init__(self, input_dim=0, output_dim=0, lr=0.01): self.input_dim = input_dim self.lr = lr # LSTM 신경망 self.model = Sequential() self.model.add(LSTM(128, input_shape=(1, input_dim), return_sequences=True, stateful=False, dropout=0.5, activation='relu')) self.model.add(BatchNormalization()) self.model.add(Attention(1)) ### ATTENTION self.model.add(Dense(32, activation='relu')) self.model.add(Dropout(0.5)) self.model.add(Dense(2, activation="softmax")) #self.model.compile(optimizer=Adam(lr=lr), loss='mse') nadam = optimizers.nadam(lr=0.01, beta_1=0.9, beta_2=0.999) self.model.compile(loss='mean_squared_error', optimizer=nadam) #self.model.compile(optimizer=Adam(lr=lr,beta_1=0.9, beta_2=0.999, amsgrad=True), loss='mse') self.prob = None
def build(self): """Builds the network using Keras. """ word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH+2), dtype="int32") inputs = [word_inputs] word_outputs = self._build_word_cnn(word_inputs) if len(self.word_vectorizers) > 0: additional_word_inputs = [kl.Input(shape=(None, input_dim), dtype="float32") for input_dim, dense_dim in self.word_vectorizers] inputs.extend(additional_word_inputs) additional_word_embeddings = [kl.Dense(dense_dim)(additional_word_inputs[i]) for i, (_, dense_dim) in enumerate(self.word_vectorizers)] word_outputs = kl.Concatenate()([word_outputs] + additional_word_embeddings) outputs, lstm_outputs = self._build_basic_network(word_outputs) compile_args = {"optimizer": ko.nadam(lr=0.002, clipnorm=5.0), "loss": "categorical_crossentropy", "metrics": ["accuracy"]} self.model_ = Model(inputs, outputs) self.model_.compile(**compile_args) if self.verbose > 0: self.model_.summary(print_fn=log.info) return self
def evaluate_model_conv(train_X, train_y, test_X, test_y, num_pixels=256, num_classes=10, learningRate=0.001): # encode targets train_y_enc = to_categorical(train_y) test_y_enc = to_categorical(test_y) # define model model = Sequential() model.add( Conv2D(64, kernel_size=(3, 3), activation='elu', input_shape=(16, 16, 1))) #model.add(Conv2D(64, (3, 3), activation='elu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dense(num_classes, activation='sigmoid')) opt = optimizers.nadam(lr=learningRate) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=opt, metrics=['accuracy']) # fit model model.fit(train_X, train_y_enc, epochs=10, batch_size=8, verbose=2) #model.fit(train_X, train_y_enc, epochs=12, batch_size=12, verbose=2) # evaluate the model _, test_acc = model.evaluate(test_X, test_y_enc, verbose=0) print(test_acc) return model, test_acc
def create_model(self): # branch1 = self.__create_sub_model() # branch2 = self.__create_sub_model() # branch3 = self.__create_sub_model() # m4 = self.__create_sub_model() model = Sequential() model.add(Merge(self.models, mode='concat')) model.add(Dense(8, kernel_initializer=initializers.random_normal(mean=0.01, stddev=0.05, seed=c.random_seed), bias_initializer='zero', kernel_regularizer=regularizers.l2(self.reg_val), activity_regularizer=regularizers.l2(self.reg_val))) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dense(1, init='normal', activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizers.nadam(lr=self.learning_rate), metrics=['accuracy']) return model
def evaluate_model_dense(train_X, train_y, test_X, test_y, num_pixels=256, num_classes=10, learningRate=0.001): # encode targets train_y_enc = to_categorical(train_y) test_y_enc = to_categorical(test_y) #have to replace with our models # define model model = Sequential() model.add( Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='elu')) model.add(Dense(128, kernel_initializer='normal', activation='elu')) model.add(Dense(64, kernel_initializer='normal', activation='relu')) model.add(Dense(32, kernel_initializer='normal', activation='relu')) model.add(Dense(16, kernel_initializer='normal', activation='relu')) model.add( Dense(num_classes, kernel_initializer='normal', activation='sigmoid')) opt = optimizers.nadam(lr=learningRate) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) # fit model model.fit(train_X, train_y_enc, epochs=12, batch_size=12, verbose=2) # evaluate the model _, test_acc = model.evaluate(test_X, test_y_enc, verbose=0) print(test_acc) return model, test_acc
def __init__(self, input_dim=0, output_dim=0, lr=0.01): self.input_dim = input_dim self.lr = lr # LSTM 신경망 self.model = Sequential() self.model.add(LSTM(256, input_shape=(1, input_dim), return_sequences=True, stateful=False, dropout=0.5)) self.model.add(BatchNormalization()) self.model.add(LSTM(256, return_sequences=True, stateful=False, dropout=0.5)) self.model.add(BatchNormalization()) self.model.add(LSTM(256, return_sequences=False, stateful=False, dropout=0.5)) self.model.add(BatchNormalization()) self.model.add(Dense(output_dim)) self.model.add(Activation('sigmoid')) #self.model.compile(optimizer=Adam(lr=lr), loss='mse') adam = optimizers.nadam(lr=0.01, beta_1=0.9, beta_2=0.999) self.model.compile(loss='mean_squared_error', optimizer=adam) #self.model.compile(optimizer=Adam(lr=lr,beta_1=0.9, beta_2=0.999, amsgrad=True), loss='mse') self.prob = None
def create_rnn_lstm(): # Add an Input Layer input_layer = layers.Input((70, )) # Add the word embedding Layer embedding_layer = layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)(input_layer) embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer) # Add the LSTM Layer lstm_layer = layers.LSTM(256)(embedding_layer) # Add the output Layers output_layer1 = layers.Dense(50, activation="sigmoid")(lstm_layer) output_layer1 = layers.Dropout(0.5)(output_layer1) output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1) # Compile the model model = models.Model(inputs=input_layer, outputs=output_layer2) model.compile(optimizer=optimizers.nadam(), loss='binary_crossentropy') return model
def fine_tune(base_model, n_class, multi_gpu_flag=False, gpus=1, method=0): if method == 0: base_model.layers.pop() x = base_model.layers[-1].output # x = base_model.output x = GlobalAveragePooling2D()(x) x = Dense(100, activation='relu')(x) x = BatchNormalization(trainable=True, axis=1)(x) x = Dropout(0.5)(x) x = Dense(50, activation='relu')(x) x = BatchNormalization(trainable=True, axis=1)(x) predictions = Dense(n_class, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) model.compile(optimizer=nadam(lr=0.00001), loss='categorical_crossentropy', metrics=['accuracy']) print('Model compiled!') if multi_gpu_flag: model = multi_gpu_model(model, gpus=gpus) return model else: return base_model
y_test = np_utils.to_categorical(y_test - 1, 3) #y_test = np_utils.to_categorical(y_test-1, 12) y_train = train[:, 0] y_train = np_utils.to_categorical(y_train - 1, 3) #y_train = np_utils.to_categorical(y_train-1, 12) input_shape = (x_test.shape[1], 1) #fpath = './weight4/weights.{epoch:02d}-{loss:.2f}-{val_loss:.2f}.hdf5' #cp_cb = ModelCheckpoint(filepath=fpath, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) sgd1 = SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True) nadam1 = nadam(lr=0.005, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004) adam1 = Adam(lr=0.005) adamax1 = Adamax(lr=0.005) adagrad1 = Adagrad(lr=0.005) adadelta1 = Adadelta(lr=0.005) rmsprop1 = RMSprop(lr=0.005) es_cb = EarlyStopping(monitor='val_loss', min_delta=0, patience=100, verbose=0, mode='auto') #1,2, -CBS
# 기본 sgd optimizer_list.append(['SGD', optimizers.SGD()]) # momentum optimizer_list.append(['Momentum', optimizers.SGD(momentum=0.9)]) # NAG optimizer_list.append((['NAG', optimizers.SGD(momentum=0.9, nesterov=True)])) # Adagrad optimizer_list.append(['Adagrad', optimizers.adagrad()]) # RMSProp optimizer_list.append(['RMSProp', optimizers.rmsprop()]) # AdaDelta optimizer_list.append(['AdaDelta', optimizers.adadelta()]) # Adam optimizer_list.append(['Adam', optimizers.adam()]) # Nadam optimizer_list.append(['Nadam', optimizers.nadam()]) score_list = [] opt_name_list = [] for optimizer_element in optimizer_list: model.compile(loss='binary_crossentropy', optimizer=optimizer_element[1], metrics=['accuracy']) # 5. 모델 학습시키기 model.fit(x_train, y_train, epochs=1500, batch_size=64) # 6. 모델 평가하기 scores = model.evaluate(x_test, y_test) score = scores[1] * 100 opt_name = optimizer_element[0]
def prepare_model(model, num_classes, batch_size, val_batch_size, max_patience, optimizer, save_path, volume_indices, data_gen_kwargs, data_augmentation_kwargs=None, learning_rate=0.001, clipnorm=10, num_outputs=1, adversarial=False, adv_weight=0.2, save_every=0, mask_to_liver=False, show_model=True, liver_only=False): if data_augmentation_kwargs is None: data_augmentation_kwargs = {} if num_outputs not in [1, 2]: raise ValueError("num_outputs must be 1 or 2") if liver_only and num_outputs != 1: raise ValueError("num_outputs must be 1 when liver_only is True") if not liver_only: lesion_output = 'output_0' liver_output = 'output_1' else: lesion_output = None liver_output = 'output_0' if adversarial and not num_outputs == 2: print("num_outputs must be 2 when adversarial is True") ''' Data generators for training and validation sets ''' gen = {} print(' > Preparing data generators...') if 'recurrent' in data_gen_kwargs: data_gen_kwargs['model'] = model gen['train'] = data_generator(volume_indices=volume_indices['train'], batch_size=batch_size, shuffle=True, loop_forever=True, transform_kwargs=data_augmentation_kwargs, **data_gen_kwargs) gen['valid'] = data_generator(volume_indices=volume_indices['valid'], batch_size=val_batch_size, shuffle=False, loop_forever=True, transform_kwargs=None, **data_gen_kwargs) gen['valid_callback'] = data_generator( \ volume_indices=volume_indices['valid'], batch_size=val_batch_size, shuffle=False, loop_forever=False, transform_kwargs=None, **data_gen_kwargs) ''' Metrics ''' metrics = {} if lesion_output: metrics[lesion_output] = [] if num_outputs == 2 or lesion_output is None: metrics[liver_output] = [] # Accuracy def accuracy(y_true, y_pred): y_true_ = K.clip(y_true - 1, 0, 1) if num_classes == 1: return K.mean(K.equal(y_true, K.round(y_pred))) else: return K.mean( K.equal(K.squeeze(y_true, 1), K.argmax(y_pred, axis=1))) if lesion_output: metrics[lesion_output].append(accuracy) # Dice averaged over slices. if lesion_output: metrics[lesion_output].append(dice_loss(2)) metrics[lesion_output].append(dice_loss(2, masked_class=0)) if num_outputs == 2 or lesion_output is None: metrics[liver_output].append(dice_loss([1, 2])) ''' Callbacks ''' callbacks = OrderedDict() ## Define early stopping callback #early_stopping = EarlyStopping(monitor='val_acc', mode='max', #patience=max_patience, verbose=0) #callbacks.append(early_stopping) # Save prediction images if save_every: save_predictions = SavePredictions(num_epochs=save_every, data_gen=gen['valid_callback'], save_path=os.path.join( save_path, "predictions")) callbacks['save_predictions'] = save_predictions # Compute dice on the full data if lesion_output: output_name = lesion_output if num_outputs == 2 else None dice_lesion = Dice(target_class=2, output_name=output_name) dice_lesion_inliver = Dice(target_class=2, mask_class=0, output_name=output_name) callbacks['dice_lesion'] = dice_lesion callbacks['dice_lesion_inliver'] = dice_lesion_inliver metrics[lesion_output].extend(dice_lesion.get_metrics()) metrics[lesion_output].extend(dice_lesion_inliver.get_metrics()) if num_outputs == 2 or lesion_output is None: output_name = liver_output if num_outputs == 2 else None dice_liver = Dice(target_class=[1, 2], output_name=output_name) callbacks['dice_liver'] = dice_liver metrics[liver_output].extend(dice_liver.get_metrics()) # Define model saving callback if lesion_output is not None: monitor = 'val_dice_loss_2' if num_outputs==1 \ else 'val_output_0_dice_loss_2' if mask_to_liver: monitor += '_m0' else: monitor = 'val_dice_loss_1_2' if num_outputs==1 \ else 'val_output_0_dice_loss_1_2' checkpointer_best_ldice = ModelCheckpoint(filepath=os.path.join( save_path, "best_weights_ldice.hdf5"), verbose=1, monitor=monitor, mode='min', save_best_only=True, save_weights_only=False) if lesion_output is not None: monitor = 'val_dice_2' if num_outputs == 1 else 'val_output_0_dice_2' if mask_to_liver: monitor += '_m0' else: monitor = 'val_dice_1_2' if num_outputs==1 \ else 'val_output_0_dice_1_2' checkpointer_best_dice = ModelCheckpoint(filepath=os.path.join( save_path, "best_weights_dice.hdf5"), verbose=1, monitor=monitor, mode='max', save_best_only=True, save_weights_only=False) if not mask_to_liver and lesion_output is not None: monitor = 'val_dice_loss_2_m0' if num_outputs==1 \ else 'val_output_0_dice_loss_2_m0' checkpointer_best_mldice = ModelCheckpoint(\ filepath=os.path.join(save_path, "best_weights_mldice.hdf5"), verbose=1, monitor=monitor, mode='min', save_best_only=True, save_weights_only=False) callbacks['checkpointer_best_mldice'] = checkpointer_best_mldice callbacks['checkpointer_best_ldice'] = checkpointer_best_ldice callbacks['checkpointer_best_dice'] = checkpointer_best_dice # Save every last epoch checkpointer_last = ModelCheckpoint(filepath=os.path.join( save_path, "weights.hdf5"), verbose=0, save_best_only=False, save_weights_only=False) callbacks['checkpointer_last'] = checkpointer_last # File logging logger = FileLogger( log_file_path=os.path.join(save_path, "training_log.txt")) callbacks['logger'] = logger ''' Compile model ''' print('\n > Compiling model...') if optimizer == 'RMSprop': optimizer = RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-8, decay=0., clipnorm=clipnorm) elif optimizer == 'nadam': optimizer = nadam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0, clipnorm=clipnorm) elif optimizer == 'adam': optimizer = adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0, clipnorm=clipnorm) elif optimizer == 'sgd': optimizer = SGD(lr=learning_rate, momentum=0.9, decay=0., nesterov=True, clipnorm=clipnorm) else: raise ValueError("Unknown optimizer: {}".format(optimizer)) if not hasattr(model, 'optimizer'): def loss(loss_func, weight): def f(y_true, y_pred): loss = loss_func(y_true, y_pred) * weight return loss f.__name__ = loss_func.__name__ return f masked_class = 0 if mask_to_liver else None losses = {} if lesion_output: losses[lesion_output] = loss( dice_loss(2, masked_class=masked_class), 1. - adv_weight) if num_outputs == 2 or lesion_output is None: losses[liver_output] = loss(dice_loss([1, 2]), 1. - adv_weight) if adversarial: losses['out_adv_0_d'] = loss(mean_squared_error, adv_weight) losses['out_adv_1_d'] = loss(mean_squared_error, adv_weight) losses['out_adv_0_g'] = loss(mean_squared_error, adv_weight) losses['out_adv_1_g'] = loss(mean_squared_error, adv_weight) losses['out_disc_0'] = loss(mean_squared_error, adv_weight) losses['out_disc_1'] = loss(mean_squared_error, adv_weight) model.compile(loss=losses, optimizer=optimizer, metrics=metrics) ''' Print model summary ''' if show_model: from keras.utils.visualize_util import plot #model.summary() plot(model, to_file=os.path.join(save_path, 'model.png')) return model, callbacks, gen
def s5(cuda,results,npydata,kfold_splits,batch_size,epochs): os.environ["CUDA_VISIBLE_DEVICES"]=cuda ogdata = np.load(npydata) num_classes = 3 if os.path.isdir(results): print('already exists') else: os.mkdir(results) ### Shuffle and parse data into x and y np.random.shuffle(ogdata) X = ogdata[:,:,:,1:] y = ogdata[:,1,1,0] y=y-1 ###KFOLDSPLITS - insert y, get test[] and train[] for different folds y = y[np.newaxis].T y = to_categorical(y, num_classes) train =[[] for i in range(kfold_splits)] test =[[] for i in range(kfold_splits)] # Go through each class for cla in range (num_classes): splitlist = [] # Grab all examples from class and shuffle shuffledclass = np.squeeze(np.nonzero(y[:, cla])) np.random.shuffle(shuffledclass) # print('class ' + str(cla) + ' ' + str(shuffledclass)) #distribute shuffled examples of class evenly into each fold for fold in range(kfold_splits): startindex=round(fold*(shuffledclass.shape[0]/kfold_splits)) endindex =round(((fold + 1) * shuffledclass.shape[0] / kfold_splits)) splitlist.append(shuffledclass[startindex:endindex]) # print(shuffledclass[startindex:endindex]) #Concatenate examples of class cla with previous examples for fold in range(kfold_splits): test[fold] = np.append(np.asarray(splitlist[fold]), test[fold]) train[fold] = np.append(np.setdiff1d(shuffledclass, np.asarray(splitlist[fold])),train[fold]) # print('fold ' + str(fold) + str(np.asarray(splitlist[fold]))) for kf in range(kfold_splits): print("Running fold", kf+1, "/", kfold_splits) ### Get indices of train/test data for current fold kf_train = train[kf].astype(int) kf_test = test[kf].astype(int) x_train, x_test = X[kf_train], X[kf_test] y_train, y_test = y[kf_train], y[kf_test] ### Check that our classes are evenly distributed in training/testing print ('ytrain0 ' + str(sum(y_train[:, 0]))) print ('ytrain1 ' + str(sum(y_train[:, 1]))) print ('ytrain2 ' + str(sum(y_train[:, 2]))) print ('ytest0 ' + str(sum(y_test[:, 0]))) print ('ytest1 ' + str(sum(y_test[:, 1]))) print ('ytest2 ' + str(sum(y_test[:, 2]))) # Save test (x,y) data x_test_loc = os.path.join(results, 'x_test' + str(kf) + '.npy') x_train_loc = os.path.join(results, 'x_train' + str(kf) + '.npy') y_test_loc = os.path.join(results, 'y_test' + str(kf) + '.npy') y_train_loc = os.path.join(results, 'y_train' + str(kf) + '.npy') np.save(x_test_loc, x_test) #np.save(x_train_loc, x_train) np.save(y_test_loc, y_test) #np.save(y_train_loc, y_train) os.environ["CUDA_VISIBLE_DEVICES"] = cuda data_augmentation = True model = Sequential() # Block 1 model.add(Conv2D(64, 3, activation='relu', padding='same', name='block1_conv1', input_shape=x_train.shape[1:])) model.add(Conv2D(64, 3, activation='relu', padding='same', name='block1_conv2')) model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.25)) # Block 2 model.add(Conv2D(128, 3, activation='relu', padding='same', name='block2_conv1')) model.add(Conv2D(128, 3, activation='relu', padding='same', name='block2_conv2')) model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.25)) # Block 3 model.add(Conv2D(256, 3, activation='relu', padding='same', name='block3_conv1')) model.add(Conv2D(256, 3, activation='relu', padding='same', name='block3_conv2')) model.add(Conv2D(256, 3, activation='relu', padding='same', name='block3_conv3')) model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.25)) # Block 4 model.add(Conv2D(512, 3, activation='relu', padding='same', name='block4_conv1')) model.add(Conv2D(512, 3, activation='relu', padding='same', name='block4_conv2')) model.add(Conv2D(512, 3, activation='relu', padding='same', name='block4_conv3')) model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.25)) model.add(Flatten()) # model.add(Dense(4096, activation='relu')) # model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) ### Choice of optimizer adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) nadam = optimizers.nadam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.0004) sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=nadam, metrics=['accuracy']) ### Data Augmentation if not data_augmentation: print('Not using data augmentation.') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True) else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.0, # randomly shift images horizontally (fraction of total width) height_shift_range=0.0, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=True, shear_range=0.1, zoom_range=0.1) # randomly flip images ###Run the model history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, validation_data=(x_test, y_test)) ###Save Results # Dump history data into history(kf).pkl file with open(os.path.join(results, 'history' + str(kf) + '.pkl'), 'wb') as f: pickle.dump(history.history, f, pickle.HIGHEST_PROTOCOL) # Save Model model.save(os.path.join(results, 'model' + str(kf) + '.h5')) K.clear_session()
model.add(Convolution2D(nb_filters2, nb_conv2, nb_conv2, border_mode='valid')) model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool))) model.add(Activation('tanh')) model.add(Convolution2D(nb_filters3, nb_conv3, nb_conv3, border_mode='valid')) #model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dense(256)) model.add(Activation('tanh')) model.add(Dropout(0.4)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) optimizer=nadam() model.compile(loss='sparse_categorical_crossentropy',optimizer=optimizer, metrics=['accuracy']) model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,verbose=1, validation_data=(X_test, Y_test)) score = model.evaluate(X_test, Y_test, verbose=0) print('Test score:', score[0]) print('Test accuracy:', score[1]) # file name to save model filename='homus_cnn.h5' # save network model model.save(filename)
def create_optimizer_instance(self, **d): return optimizers.nadam(**d)
from keras.optimizers import nadam from keras.optimizers import SGD,Adadelta import keras.optimizers from keras.wrappers.scikit_learn import KerasRegressor model=Sequential() model.add(Dense(3,activation='selu')) model.add(Dense(5,activation='selu')) model.add(Dense(7,activation='relu')) model.add(Dense(11,activation='selu')) model.add(Dense(5,activation='selu')) #model.add(Dense(3,activation='selu')) model.add(Dense(1,kernel_initializer='normal')) nadam_mode=nadam(lr=0.01) sgd=SGD( lr=0.001,momentum=0.3,decay=0.2) adaDelta=Adadelta(lr=0.01, rho=0.95, epsilon=None, decay=0.0) model.compile(loss='mean_squared_error', optimizer=adaDelta) history=model.fit(x=X.values,y=y.values,batch_size=16,epochs=50,verbose=1) plt.plot(history.history['loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'validation'], loc='upper left') plt.show() test=pd.read_csv('Income_testing.csv') test=test.drop(['ID'],axis=1)
x=layers.Dense(1,kernel_initializer='normal')(x) model=keras.Model(inputs=inlayer, outputs=x) # y=layers.Dense(300,activation='tanh')(inlayer) # y=layers.Dense(160,activation='tanh')(y) # y=layers.Dense(70,activation='tanh')(y) # m =layers.concatenate([x, y]) # m=layers.Dense(350,activation='tanh')(m) # m=layers.Dense(350,activation='tanh')(m) # m=layers.Dense(768,activation='tanh')(m) # m=layers.Dense(1,kernel_initializer='normal')(m) # model=keras.Model(inputs=inlayer, outputs=m) print(model.summary()) model.compile(loss='mean_squared_error',optimizer=nadam(lr=5e-5),metrics=['mse']) best_model=model.fit(np.array(train_x),column_or_1d(train_y),epochs=500,batch_size=32,validation_split=0.2,callbacks=[ EarlyStopping(monitor='val_loss',min_delta=0,patience=16,verbose=1,mode='auto',baseline=None,restore_best_weights=True) ]) test_y=pd.DataFrame(model.predict(test_x),index=test_x.index,columns=["SalePrice"]) test_y=reverse(test_y) test_y.to_csv(f"submission_{type(model).__name__}.csv",index=True,index_label="Id") run_sklearn=True if run_sklearn: scoring='neg_mean_squared_error' models=[ # DecisionTreeRegressor(), # KNeighborsRegressor(), # GradientBoostingRegressor(), # AdaBoostRegressor(),
x=model.layers[7].output #take the first 5 layers of the model x=Flatten()(x) x=Dense(1024, activation="relu")(x) x=Dropout(0.5)(x) x=Dense(384, activation="relu")(x) x=Dropout(0.5)(x) x=Dense(96, activation="relu")(x) x=Dropout(0.5)(x) predictions = Dense(30, activation="softmax")(x) model_final =Model(input=model.input, output=predictions) #model_final = load_model("weights_Mobile_Net.h5") model_final.compile(loss="categorical_crossentropy", optimizer=optimizers.nadam(lr=0.00001), metrics=["accuracy"]) train_datagen = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2, horizontal_flip = True, fill_mode="nearest", width_shift_range=0.3, height_shift_range=0.3, rotation_range=30) test_datagen = ImageDataGenerator(rescale = 1./255, horizontal_flip = True, fill_mode = "nearest", zoom_range = 0.3, width_shift_range = 0.3,
import numpy as np from sklearn.model_selection import train_test_split # pickle_in = open("cooked/data.pickle","rb") # x_train, y_train = pickle.load(pickle_in) # x_train = x_train.reshape(-1,120,1) # x_train /= 3000 # y_train /= 4000 # # x_train, x_val, y_train, y_val = x_train[:10000], x_train[10000:], y_train[:10000], y_train[10000:] model = Sequential() model.add( Convolution1D(16, kernel_size=11, strides=4, activation=elu, input_shape=(120, 1))) model.add(Reshape((16, 28))) model.add(LSTM(16, activation=sigmoid)) model.add(Dense(1)) model.compile(optimizer=nadam(), loss=mean_absolute_error) # def train_for_epochs(n): # for i in range(n): # x_noise = np.random.normal(scale=0.05, size=x_train.shape) # y_noise = np.random.normal(scale=0.05, size=y_train.shape) # model.fit(x_train + x_noise, y_train + y_noise, epochs=3) # print(model.evaluate(x_val, y_val)) # model.save("models/cnn")