def get_optimizer(op_type, learning_rate): if op_type == 'sgd': return optimizers.SGD(learning_rate) elif op_type == 'rmsprop': return optimizers.RMSprop(learning_rate) elif op_type == 'adagrad': return optimizers.Adagrad(learning_rate) elif op_type == 'adadelta': return optimizers.Adadelta(learning_rate) elif op_type == 'adam': return optimizers.Adam(learning_rate, clipnorm=5) elif op_type == 'adamw': return AdamWeightDecay( learning_rate=learning_rate, weight_decay_rate=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-6, exclude_from_weight_decay=["layer_norm", "bias"]) elif op_type == 'adamw_2': return create_optimizer(init_lr=learning_rate, num_train_steps=9000, num_warmup_steps=0) elif op_type == 'adamw_3': return create_optimizer(init_lr=learning_rate, num_train_steps=9000, num_warmup_steps=100) else: raise ValueError('Optimizer Not Understood: {}'.format(op_type))
def __init__(self, path: str, epochs: int = 50, model: str or tf.keras.models.Sequential = None, optimizer: tko.Optimizer = tko.Adadelta(lr=0.1), threshold: float = None, verbose: bool = False): """ Initialize audio downloader :param path: working path :param epochs: max training epochs :param model: path to existing or keras Sequential :param optimizer: keras optimizer :param threshold: stop training if threshold exceeds given value :param verbose: verbose output """ self.path = path self.model = model self.optimizer = optimizer self.verbose = verbose self.data = {} self.model = model self.threshold = threshold self.epochs = epochs
def train(): global model, x_train, y_train model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', padding="same", input_shape=(imgW, imgH, 1))) model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', padding="same")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', padding="same")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(classes_num, activation='softmax')) model.summary() print() model.compile(loss=categorical_crossentropy, optimizer=optimizers.Adadelta(), metrics=['accuracy']) model.fit(x=x_train, y=y_train, batch_size=batch_size, verbose=2, epochs=epochs, validation_split=0.2) model.save(filepath=modelfile)
def descent_optimizers(): optimizers.Adadelta(learning_rate=1e-3, rho=0.95, epsilon=1e-07, name='Adadelta') optimizers.Adagrad(learning_rate=1e-3, initial_accumulator_value=0.1, epsilon=1e-07, name='Adagrad') optimizers.Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam') optimizers.Adamax(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax') optimizers.Nadam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam') optimizers.RMSprop(learning_rate=1e-3, rho=0.9, momentum=0.0, epsilon=1e-07, centered=False, name='RMSprop') optimizers.SGD(learning_rate=1e-2, momentum=0.0, nesterov=False, name='SGD')
def main(): #file = r'./db/fucDatasetReg_1F_NoLinear.csv' #file = r'./db/fucDatasetReg_2F.csv' file = r'./db/fucDatasetReg_3F_1000.csv' x_train, x_test, y_train, y_test = getCsvDataset(file) lr = 1e-3 EPOCHES = 200 # optimizer = optimizerTf(lr=lr) # losses,_ = trainModel(x_train,y_train,optimizer,epochs=EPOCHES) # plotLoss(losses) opts = [] # fast group opts.append((optimizers.SGD(learning_rate=lr), 'SGD')) opts.append((optimizers.RMSprop(learning_rate=lr), 'RMSprop')) opts.append((optimizers.Adam(learning_rate=lr), 'Adam')) opts.append((optimizers.Adamax(learning_rate=lr), 'Adamax')) opts.append((optimizers.Nadam(learning_rate=lr), 'Nadam')) # # slow group opts.append((optimizers.Adadelta(learning_rate=lr), 'Adadelta')) opts.append((optimizers.Adagrad(learning_rate=lr), 'Adagrad')) opts.append((optimizers.Ftrl(learning_rate=lr), 'Ftrl')) lossesDict = {} for opti, name in opts: losses, _ = trainModel(x_train, y_train, opti, epochs=EPOCHES) lossesDict[name] = losses #print(name, losses) plotLossDict(lossesDict)
def build_optimizer(type, lr, kerasDefaults): """ Set the optimizer to the appropriate Keras optimizer function based on the input string and learning rate. Other required values are set to the Keras default values Parameters ---------- type : string String to choose the optimizer Options recognized: 'sgd', 'rmsprop', 'adagrad', adadelta', 'adam' See the Keras documentation for a full description of the options lr : float Learning rate kerasDefaults : list List of default parameter values to ensure consistency between frameworks Returns ---------- The appropriate Keras optimizer function """ if type == 'sgd': return optimizers.SGD(lr=lr, decay=kerasDefaults['decay_lr'], momentum=kerasDefaults['momentum_sgd'], nesterov=kerasDefaults['nesterov_sgd']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'rmsprop': return optimizers.RMSprop(lr=lr, rho=kerasDefaults['rho'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adagrad': return optimizers.Adagrad(lr=lr, epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adadelta': return optimizers.Adadelta(lr=lr, rho=kerasDefaults['rho'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adam': return optimizers.Adam(lr=lr, beta_1=kerasDefaults['beta_1'], beta_2=kerasDefaults['beta_2'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # ,
def get_keras_optimizer(optimizer_params: Dict): from tensorflow.keras import optimizers #pylint: disable=import-error if optimizer_params['type'] == 'sgd': return optimizers.SGD(**optimizer_params['kwargs']) if optimizer_params['type'] == 'adadelta': return optimizers.Adadelta(**optimizer_params['kwargs']) if optimizer_params['type'] == 'adam': return optimizers.Adam(**optimizer_params['kwargs']) raise ValueError('The optimizer {} is not supported.'.format( optimizer_params['type']))
def callOptimizer(opt='rmsprop'): '''Function returns the optimizer to use in .fit() options: adam, sgd, rmsprop, ada_grad,ada_delta,ada_max ''' opt_dict = {'adam': optimizers.Adam(), 'sgd' : optimizers.SGD(), 'rmsprop' : optimizers.RMSprop(), 'ada_grad' : optimizers.Adagrad(), 'ada_delta': optimizers.Adadelta(), 'ada_max' : optimizers.Adamax()} return opt_dict[opt]
def definePacmanTestModel1(conf): # Define Model inputShape = (conf.input_y_dim, conf.input_x_dim, conf.c_channels) state = Input(inputShape) # pre 0 in x = Conv2D(32, (3, 3), activation='relu', padding='same', name='Conv0')(state) # conv 0 x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv1')(x) # conv 1 x = MaxPooling2D((2, 2))(x) # pooling x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv2')(x) # conv 2 x = MaxPooling2D((2, 2))(x) # pooling x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv3')(x) # conv 3 x = MaxPooling2D((2, 2))(x) # pooling x = Flatten()(x) # flatten x = Dense(100, activation='relu')(x) # fc x = Dense(100, activation='relu')(x) # fc qsa = Dense(conf.num_actions, activation='linear')(x) # out # Make Model model = Model(state, qsa) # Configure Optimizer if conf.optimizer == 'adadelta': optimizer = optimizers.Adadelta(lr=conf.learning_rate, decay=0.0, rho=0.95) elif conf.optimizer == 'sgd': optimizer = optimizers.SGD(lr=conf.learning_rate) elif conf.optimizer == 'adam': optimizer = optimizers.Adam(lr=conf.learning_rate) elif conf.optimizer == 'adagrad': optimizer = optimizers.Adagrad(lr=conf.learning_rate) else: print("Optimizer '{0}' not found.".format(conf.optimizer)) exit(0) return model, optimizer
def create_optimizer(opt, learning_rate, momentum=0.9, decay=0.0, nesterov=False): """ Create optimizer operation :param opt: A string which can be one of 'sgd', 'momentum' or 'adam' :param learning_rate: A float value :param momentum: A float value :return: An optimizer operation """ assert opt in [ 'sgd', 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax', 'nadam' ] if opt == 'sgd': optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum, decay=decay, nesterov=nesterov) elif opt == 'rmsprop': optimizer = optimizers.RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-06) elif opt == 'adagrad': optimizer = optimizers.Adagrad(lr=learning_rate, epsilon=1e-06) elif opt == 'adadelta': optimizer = optimizers.Adadelta(lr=learning_rate, rho=0.95, epsilon=1e-06) elif opt == 'adam': optimizer = optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'adamax': optimizer = optimizers.Adamax(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'nadam': optimizer = optimizers.Nadam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) else: optimizer = None return optimizer
def deep_network(self, pca=None): if not pca: print("Regular") model = Sequential() model.add(Dense(64, activation='relu', input_dim=self.train_x_.shape[1])) model.add(Dense(16, activation='relu')) model.add(Dense(self.classes, activation='relu')) opt = optimizers.Adadelta(learning_rate=1.0, rho=0.95) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) print("samples: ", self.train_x_.shape, "classes: ", self.train_y_cat.shape) model.fit(self.train_x_, self.train_y_cat, validation_split=0.2, batch_size=32, epochs=50) model.evaluate(self.test_x_, self.test_y_cat) # model_prediction = model.predict(self.test_x_, batch_size=32, verbose=1) # # return model_prediction elif pca: print("PCA") model_pca = Sequential() model_pca.add(Dense(64, activation='tanh', input_dim=self.train_x_pca_.shape[1])) model_pca.add(Dropout(0.25)) model_pca.add(Dense(16, activation='tanh')) model_pca.add(Dense(self.classes, activation='softmax')) opt = optimizers.Adadelta(learning_rate=1.0, rho=0.95) model_pca.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model_pca.fit(self.train_x_pca_, self.train_y_cat, validation_split=0.2, batch_size=32, epochs=10) model_pca.evaluate(self.test_x_pca_, self.test_y_cat)
def __get_optimizer(optimizer, lr): if optimizer == 'sgd': return optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': return optimizers.RMSprop(lr=lr) elif optimizer == 'adagrad': return optimizers.Adagrad(lr=lr) elif optimizer == 'adadelta': return optimizers.Adadelta(lr=lr) elif optimizer == 'adam': return optimizers.Adam(lr=lr) elif optimizer == 'adamax': return optimizers.Adamax(lr=lr) elif optimizer == 'nadam': return optimizers.Nadam(lr=lr)
def _compile(self, model, loss_function, optimizer, lr=0.01, decay=0.0, clipnorm=0.0): """Compiles a model specified with Keras. See https://keras.io/optimizers/ for more info on each optimizer. Args: model: Keras model object to compile loss_function: Keras loss_function object to compile model with optimizer (str): the optimizer to use during training lr (float): learning rate to use during training decay (float): per epoch decay rate clipnorm (float): gradient normalization threshold """ # The parameters of these optimizers can be freely tuned. if optimizer == 'sgd': optimizer_ = optimizers.SGD(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adam': optimizer_ = optimizers.Adam(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adamax': optimizer_ = optimizers.Adamax(lr=lr, decay=decay, clipnorm=clipnorm) # It is recommended to leave the parameters of this optimizer at their # default values (except the learning rate, which can be freely tuned). # This optimizer is usually a good choice for recurrent neural networks elif optimizer == 'rmsprop': optimizer_ = optimizers.RMSprop(lr=lr, clipnorm=clipnorm) # It is recommended to leave the parameters of these optimizers at their # default values. elif optimizer == 'adagrad': optimizer_ = optimizers.Adagrad(clipnorm=clipnorm) elif optimizer == 'adadelta': optimizer_ = optimizers.Adadelta(clipnorm=clipnorm) elif optimizer == 'nadam': optimizer_ = optimizers.Nadam(clipnorm=clipnorm) else: err_msg = "Argument for `optimizer` is invalid, got: {}".format( optimizer) LOGGER.error('ValueError %s', err_msg) raise ValueError(err_msg) model.compile(optimizer=optimizer_, loss=loss_function)
def construct_model(): input1 = Input(shape=(seq_length, 5790, 6)) input2 = Input(shape=(seq_length, 6)) re_input1 = Reshape((5790 * seq_length, 6))(input1) rere_input1 = Permute((2, 1), input_shape=(5790 * seq_length, 6))(re_input1) conv1 = Conv1D(30, 1, strides=1, padding='valid', activation='relu', data_format="channels_first", name='X1_input')( rere_input1) conv2 = Conv1D(30, 1, strides=1, padding='valid', activation='relu', data_format="channels_first", name='Conv7')( conv1) LSTM1 = LSTM(4, return_sequences=True)(input2) LSTM2 = LSTM(4, return_sequences=False)(LSTM1) reshape_conv2 = Reshape((30, 5790, seq_length))(conv2) pool = MaxPooling2D(pool_size=(1, 2), strides=(1, 2), padding='valid', data_format="channels_last")(reshape_conv2) reshape1 = Reshape((1, 30, 2895, seq_length))(pool) reshape2 = Permute((4, 2, 3, 1), input_shape=(1, 30, 2895, seq_length))(reshape1) convLSTM1 = ConvLSTM2D(filters=10, kernel_size=(3, 3), strides=(3, 3), padding='same', return_sequences=True)(reshape2) convLSTM2 = ConvLSTM2D(filters=20, kernel_size=(3, 2), strides=(2, 2), padding='same', return_sequences=True)(convLSTM1) convLSTM3 = ConvLSTM2D(filters=40, kernel_size=(3, 1), strides=(2, 2), padding='same', return_sequences=True)(convLSTM2) convLSTM4 = ConvLSTM2D(filters=40, kernel_size=(2, 2), strides=(2, 2), padding='same', return_sequences=False)(convLSTM3) flat1 = Flatten()(convLSTM4) flat2 = Flatten()(LSTM2) dense1 = Dense(120)(flat1) activation1 = Activation('relu')(dense1) merge2 = concatenate([activation1, flat2]) dense2 = Dense(30)(merge2) activation2 = Activation('relu')(dense2) output = Dense(1, kernel_regularizer=regularizers.l2(0.000001))(activation2) model = Model(inputs=[input1, input2], outputs=[output]) sgd = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0) model = multi_gpu_model(model, gpus=2) model.compile(loss='mean_squared_error', optimizer=sgd) print(model.summary()) return model
def get_optimizer(args): clipvalue = 0 clipnorm = 10 if args.algorithm == 'rmsprop': optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'sgd': optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adagrad': optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adadelta': optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adam': optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adamax': optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) else: raise Exception("Can't find optimizer " + args.algorithm) return optimizer
def get_optimizer(self): """ Returns tf.keras.optimizer based on config :return: optimizer """ if self.optimizer == 'Adam': return opt.Adam(learning_rate=self.learning_rate) elif self.optimizer == 'Adadelta': return opt.Adadelta(learning_rate=self.learning_rate) elif self.optimizer == 'RMSprop': return opt.RMSprop(learning_rate=self.learning_rate) elif self.optimizer == 'SGD': return opt.SGD(learning_rate=self.learning_rate, momentum=self.config.sgd_momentum) else: raise ValueError("%s optimizer not found in tf.keras.optimizers" % self.optimizer)
def autoencoder_stacked(entrada, train_set, test_set, dim_latente=250, epochs=125, lr=0.30, grabar=False, cargar=False, archivo_encoder=" ", archivo_autoencoder=" ", grafica=False): if cargar: encoder = load_model(archivo_encoder) autoencoder_deep = load_model(archivo_autoencoder) else: dim_input = entrada.shape[1] adadelta = optimizers.Adadelta(lr=lr, rho=0.95) input = Input(shape=((dim_input, ))) encoded = Dense(2000, activation='selu')(input) encoded = Dense(1000, activation='selu')(encoded) encoded = Dense(dim_latente, activation='selu')(encoded) decoded = Dense(1000, activation='selu')(encoded) decoded = Dense(2000, activation='selu')(decoded) decoded = Dense(dim_input, activation='sigmoid')(decoded) autoencoder_deep = Model(input, decoded) encoder = Model(input, encoded) autoencoder_deep.compile(optimizer=adadelta, loss='binary_crossentropy', metrics=["accuracy"]) autoencoder_deep.fit(train_set, train_set, epochs=epochs, batch_size=256, shuffle=True, validation_data=(test_set, test_set)) if grafica: galassify_grafica_error.grafica_loss(autoencoder_deep, epochs) if grabar: autoencoder_deep.save(archivo_autoencoder) encoder.save(archivo_encoder) return encoder, autoencoder_deep
def set_optimizers(self, select_optimizer, select_lr): print("optimizers setting.") print("optimizers : ", select_optimizer) # Configure the optimizer from config. if select_optimizer == "adam" or select_optimizer == "Adam": opt = optimizers.Adam(lr=select_lr) elif select_optimizer == "sgd" or select_optimizer == "SGD": opt = optimizers.SGD(lr=select_lr) elif select_optimizer == "adagrad" or select_optimizer == "Adagrad": opt = optimizers.Adagrad(lr=select_lr) elif select_optimizer == "adadelta" or select_optimizer == "Adadelta": opt = optimizers.Adadelta(lr=select_lr) else: print("This is an unconfigured optimizer that uses Adam instead.") opt = optimizers.Adam(lr=select_lr) print("optimizer setting ... ok.") return opt
def cc_optimizer(self, learning_rate, decay_rate=0, optimizer='adam'): if optimizer == 'sgd': self.cc_optimizer = optimizers.SGD(lr=learning_rate,\ decay = decay_rate, \ momentum = moment, \ nesterov=True) elif optimizer == 'rms': #-------------------------------------------------------------- self.cc_optimizer = optimizers.RMSprop(lr = learning_rate, \ rho= 0.9, \ epsilon = None,\ decay = decay_rate) elif optimizer == 'adagrad': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adagrad (lr = learning_rate , \ epsilon = None , \ decay = decay_rate) elif optimizer == 'adadelta': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adadelta(lr = learning_rate, \ rho=0.95 , \ epsilon = None,\ decay = decay_rate) elif optimizer == 'nadam': self.cc_optimizer = optimizers.Nadam(lr = learning_rate, \ beta_1 = 0.9, \ beta_2 = 0.999, \ epsilon = None, \ schedule_decay = 0.004) else: self.cc_optimizer = optimizers.Adam(lr = learning_rate, \ beta_1 = 0.9 , \ beta_2 = 0.999 , \ epsilon = None,\ decay = decay_rate,\ amsgrad = True ) return self.cc_optimizer
def set_optimizer(self, optimizer, loss="categorical_crossentropy"): if optimizer.lower() == "adam": opt_handle = Opt.Adam() elif optimizer.lower() == "adagrad": opt_handle = Opt.Adagrad() elif optimizer.lower() == "adadelta": opt_handle = Opt.Adadelta() elif optimizer.lower() == "rmsprop": opt_handle = Opt.RMSprop() else: print("Unknown optimizer {}. Using Adam!".format(optimizer)) opt_handle = Opt.Adam() print("Setting model optimizer to {}".format(optimizer)) self.model.compile( loss="categorical_crossentropy", optimizer=opt_handle, metrics=["accuracy"], )
def setOptimizer(self, config): configOptimizer = config["model"]["optimizer"].lower() if configOptimizer == "Adadelta".lower(): self.optimizer = optimizers.Adadelta() elif configOptimizer == "Adagrad".lower(): self.optimizer = optimizers.Adagrad() elif configOptimizer == "Adamax".lower(): self.optimizer = optimizers.Adamax() elif configOptimizer == "Ftrl".lower(): self.optimizer = optimizers.Ftrl() elif configOptimizer == "SGD".lower(): self.optimizer = optimizers.SGD() elif configOptimizer == "Nadam".lower(): self.optimizer = optimizers.Nadam() elif configOptimizer == "Optimizer".lower(): self.optimizer = optimizers.Optimizer() elif configOptimizer == "RMSprop".lower(): self.optimizer = optimizers.RMSprop()
def create_model(jets_dim, train_var, FC_layers, lr, beta, lamb, seed, encoder, n_gpus): if len(set(train_var)-{'constituents'}) == 0: input_dim = jets_dim elif 'constituents' not in train_var : input_dim = len(train_var) else : input_dim = jets_dim+len(train_var)-1 tf.debugging.set_log_device_placement(False) strategy = tf.distribute.MirroredStrategy(devices=['/gpu:'+str(n) for n in np.arange(n_gpus)]) with strategy.scope(): #loss = 'mean_squared_error' loss = 'binary_crossentropy' if encoder == 'dual_ae': model = dual_AE(jets_dim, scalars_dim, FC_layers) model.compile(optimizer=optimizers.Adadelta(lr=lr), loss=loss, loss_weights=[1.0,1.0]) else: if encoder == 'oe_vae': model = FC_VAE(input_dim, FC_layers, beta, lamb, seed) if encoder == 'vae' : model = FC_VAE(input_dim, FC_layers, beta, seed) if encoder == 'ae' : model = FC_AE (input_dim, FC_layers) model.compile(optimizer=optimizers.Adam(lr=lr, amsgrad=False), loss=loss) #model.compile(optimizer=optimizers.Adam(lr=lr, amsgrad=False), loss=loss, loss_weights=[1.0,1.0]) print('\nNEURAL NETWORK ARCHITECTURE'); model.summary(); print() return model
def get_optimizer(): optimizer_name = optimizer_names[random.randint(0, len(optimizer_names) - 1)] model_attributes.optimizer_name = optimizer_name if optimizer_name == 'SGD': return optimizers.SGD(lr=get_learning_rate()) elif optimizer_name == 'RMSprop': return optimizers.RMSprop(lr=get_learning_rate()) elif optimizer_name == 'Adagrad': return optimizers.Adagrad(lr=get_learning_rate()) elif optimizer_name == 'Adadelta': return optimizers.Adadelta(lr=get_learning_rate()) elif optimizer_name == 'Adam': return optimizers.Adam(lr=get_learning_rate()) elif optimizer_name == 'Adamax': return optimizers.Adamax(lr=get_learning_rate()) elif optimizer_name == 'Nadam': return optimizers.Nadam(lr=get_learning_rate()) return None
def create_optimizer(optimizer, learning_rate): """ Simply returns an optimizer based on the string refering it. :param optimizer: Name of the optimizer :param learning_rate: learning rate of the optimizer :return: tensorflow.keras.{optimizers}. """ if optimizer.lower() == "adadelta": return optimizers.Adadelta(lr=learning_rate) elif optimizer.lower() == "adagrad": return optimizers.Adagrad(lr=learning_rate) elif optimizer.lower() == "adam": return optimizers.Adam(lr=learning_rate) elif optimizer.lower() == "adamax": return optimizers.Adamax(lr=learning_rate) elif optimizer.lower() == "nadam": return optimizers.Nadam(lr=learning_rate) elif optimizer.lower() == "rmsprop": return optimizers.RMSprop(lr=learning_rate) elif optimizer.lower() == "sgd": return optimizers.SGD(lr=learning_rate)
def select_optimizer(optimizer: str, learning_rate: float, clipnorm: float) -> object: """Sets the optimizer for the training process of the neural network using the specified parameters. Input :param optimizer: name of the optimizer algorithm to be used (options: ADAM, SGD, RMSPROP, ADADELTA, Nesterov-ADAM) :type optimizer: str :param learning_rate: initial learning rate of the optimzier for neural network training :type learning_rate: float :param clipnorm: gradients will be clipped when their L2 norm exceeds this value. :type clipnorm: float Output :return: abstract optimizer base class of the keras package :rtype: object """ if optimizer == 'ADAM': optimizer = optimizers.Adam( lr=learning_rate, clipnorm=clipnorm) # , beta_1=0.9, beta_2=0.999) if optimizer == 'SGD': optimizer = optimizers.SGD(lr=learning_rate, nesterov=True, clipnorm=clipnorm) if optimizer == 'RMSPROP': optimizer = optimizers.RMSprop(lr=learning_rate, momentum=0.9, clipnorm=clipnorm) if optimizer == 'ADADELTA': optimizer = optimizers.Adadelta(lr=learning_rate) # , rho=0.95) if optimizer == 'Nesterov-ADAM': optimizer = optimizers.Nadam( lr=learning_rate, clipnorm=clipnorm) # , beta_1=0.91, beta_2=0.997) return optimizer
def chexnet_model(FLAGS): """ Builds the chexnet model using specifics from FLAGS. Returns a compiled model.""" base_model = DenseNet121(include_top=False, weights='imagenet', input_shape=(FLAGS.image_size, FLAGS.image_size, 3)) x = base_model.output x = GlobalAveragePooling2D()(x) predictions = Dense(14, activation='sigmoid', bias_initializer='ones')(x) model = Model(inputs=base_model.input, outputs=predictions) if FLAGS.opt == 'adam': opt = optimizers.Adam(lr=FLAGS.lr) elif FLAGS.opt == 'sgd': opt = optimizers.SGD(lr=FLAGS.lr, momentum=FLAGS.momentum, nesterov=FLAGS.nesterov) elif FLAGS.opt == 'rmsprop': opt = optimizers.RMSProp(lr=FLAGS.lr) elif FLAGS.opt == 'adagrad': opt = optimizers.Adagrad(lr=FLAGS.lr) elif FLAGS.opt == 'adadelta': opt = optimizers.Adadelta(lr=FLAGS.lr) elif FLAGS.opt == 'adamax': opt = optimizers.Adamax(lr=FLAGS.lr) elif FLAGS.opt == 'nadam': opt = optimizers.Nadam(lr=FLAGS.lr) else: print("No optimizer selected. Using Adam.") opt = optimizers.Adam(lr=FLAGS.lr) hvd_opt = hvd.DistributedOptimizer(opt) model.compile(loss='binary_crossentropy', optimizer=hvd_opt, metrics=['accuracy']) return model
def get_optimizer(optimization_function, learning_rate): if optimization_function == "Adam": optimization_function = Optimizer.Adam(learning_rate=learning_rate) elif optimization_function == "SGD": optimization_function = Optimizer.SGD(learning_rate=learning_rate) elif optimization_function == "RMSprop": optimization_function = Optimizer.RMSprop(learning_rate=learning_rate) elif optimization_function == "Adagrad": optimization_function = Optimizer.Adagrad(learning_rate=learning_rate) elif optimization_function == "Adadelta": optimization_function = Optimizer.Adadelta(learning_rate=learning_rate) elif optimization_function == "Adamax": optimization_function = Optimizer.Adamax(learning_rate=learning_rate) elif optimization_function == "Nadam": optimization_function = Optimizer.Nadam(learning_rate=learning_rate) return optimization_function
print(y_train.shape) print(x_train.shape) model = Sequential() #更正:不是model.Sequential() input_shape = [im_row, im_col, 1] model.add( Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(Conv2D(64, kernel_size=(3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) #更正:这里是pool_size 不是kernel_size model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.25)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=losses.categorical_crossentropy, optimizer=optimizers.Adadelta(), metrics=['accuracy']) #这里的optimizers 有s history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_data=(x_test, y_test)) #注意是epochs 不是es # plt.plot(history.history['loss']) # plt.plot(history.history['val_loss']) # plt.title('Model loss') # plt.ylabel('loss') # plt.xlabel('Epoch') # plt.legend(['Train', 'Test'], loc='upper left') # plt.show()
print(Utils.classificationReport('test', db.y_test, y_pred)) Utils.rocCurve( db.name + '_' + schema.name + '_' + estimator.name + '_Augmented', db.Y_test(), estimator.predict(db.X_test), db.info['n_cls']) schema.saveWeights(estimator.name + '_' + db.name + '_Augmented') schema.extract(estimator.name + '_Augmented', db) for data, version, augment in Conf: db = Loader.getDataset(data) db = Utils.preProcessing(db) # db.histogram() db.summary() # %% Conventional # NotAugmentedRun(db, version, Estm.Conventional, # losses.categorical_crossentropy, # optimizers.Adadelta(), ['acc'], # [callbacks.EarlyStopping(patience=20)]) generator = General(X_train=db.X_train, y_train=db.y_train, augment=True, allowable=augment) AugmentedRun(db, generator, version, Estm.Conventional, losses.categorical_crossentropy, optimizers.Adadelta(), ['acc'], [callbacks.EarlyStopping(patience=20)])
def train_ae_coea(net_params, layer_params_list, data_train, data_eval, n_layers=4, iters=4000, ind=None): """Trains the Autoencoder with the given parameters. Returns max Rho_MK and validation loss""" K.clear_session() # Required in order to have reproducible results from a specific random seed os.environ['PYTHONHASHSEED'] = '0' # Force tf to use a single thread (required for reproducibility) session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf) tf.compat.v1.keras.backend.set_session(sess) # network parameters batch = 32 optim = net_params['optim'] learn_rate = net_params['learn_rate'] decay = net_params['decay'] mom = net_params['mom'] rand_seed = net_params['rand_seed'] np.random.seed(rand_seed) rn.seed(rand_seed) tf.compat.v1.set_random_seed(rand_seed) iter_per_epoch = int(np.ceil(len(data_train) / batch)) epochs = int(np.ceil(iters / iter_per_epoch)) if optim == 'adam': opt = optimizers.Adam(lr=learn_rate, beta_1=mom, decay=decay, clipvalue=0.3) elif optim == 'nadam': opt = optimizers.Nadam(lr=learn_rate, beta_1=mom, schedule_decay=decay, clipvalue=0.3) elif optim == 'rmsprop': opt = optimizers.RMSprop(lr=learn_rate, rho=mom, decay=decay, clipvalue=0.3) else: # adadelta opt = optimizers.Adadelta(lr=learn_rate, rho=mom, decay=decay, clipvalue=0.3) # pretraining input_data = data_train weights = [] ea = EarlyStopping(patience=int(epochs / 3)) cb = [ea] for i, layer_params in enumerate(layer_params_list): layer_params = layer_params_list[n_layers - 1 - i] input_layer = layers.Input(shape=(input_data.shape[1],)) hidden_layer = layers.Dense(layer_params['n_neuron'], activation=layer_params['act'], kernel_regularizer=regularizers.l2(layer_params['L2']), activity_regularizer=SparseActivityRegularizer (p=layer_params['SP'], sparsity_beta=layer_params['SR']), kernel_initializer=layer_params['init'])(input_layer) output_layer = layers.Dense(input_data.shape[1], activation=layer_params['act'], kernel_initializer=layer_params['init'])(hidden_layer) model = models.Model(input_layer, output_layer) model.compile(optimizer=opt, loss='mse') history = model.fit(x=input_data, y=input_data, batch_size=batch, epochs=epochs, callbacks=cb, validation_split=0.2, verbose=False) for loss in history.history['loss']: if np.isnan(loss): K.clear_session() return 0.01, 100 h_weights = model.get_weights() weights.insert(2 * i, h_weights[0]) weights.insert(2 * i + 1, h_weights[1]) weights.insert(len(weights) - 2 * i, h_weights[2]) weights.insert(len(weights) - 2 * i, h_weights[3]) model = models.Model(input_layer, hidden_layer) input_data = model.predict(input_data) # stacking the layers - fine tuning input_layer = layers.Input(shape=(data_train.shape[1],)) enc = layers.Dense(layer_params_list[-1]['n_neuron'], activation=layer_params_list[-1]['act'], kernel_initializer=layer_params_list[-1]['init'])(input_layer) for i in range(n_layers - 1): enc = layers.Dense(layer_params_list[-2 - i]['n_neuron'], activation=layer_params_list[-2 - i]['act'], kernel_initializer=layer_params_list[-2 - i]['init'])(enc) dec = layers.Dense(layer_params_list[1]['n_neuron'], activation=layer_params_list[1]['act'], kernel_initializer=layer_params_list[1]['init'])(enc) for i in range(n_layers - 2): dec = layers.Dense(layer_params_list[i + 2]['n_neuron'], activation=layer_params_list[i + 2]['act'], kernel_initializer=layer_params_list[i + 2]['init'])(dec) output_layer = layers.Dense(len(data_train.T), activation=layer_params_list[-1]['act'], kernel_initializer=layer_params_list[-1]['init'])(dec) # assumption: output layer has the same parameters as the final hidden layer model = models.Model(input_layer, output_layer) model.compile(optimizer=opt, loss='mse') model.set_weights(weights) history = model.fit(x=data_train, y=data_train, batch_size=batch, epochs=epochs, callbacks=cb, validation_data=(data_eval, data_eval), verbose=False) if ind: ind.final_weights = model.get_weights() for loss in history.history['loss']: if np.isnan(loss): K.clear_session() return 0.01, 100 val_loss = history.history['val_loss'][-1] model = models.Model(input_layer, enc) indicators = model.predict(data_eval) # MK test rho_mk = calc_rho(indicators) max_rho_mk = 0.01 if np.isnan(max(abs(rho_mk))) else max(abs(rho_mk)) loss = 100 if np.isnan(val_loss) else val_loss return max_rho_mk, loss