def get_optimizer(op_type, learning_rate): if op_type == 'sgd': return optimizers.SGD(learning_rate) elif op_type == 'rmsprop': return optimizers.RMSprop(learning_rate) elif op_type == 'adagrad': return optimizers.Adagrad(learning_rate) elif op_type == 'adadelta': return optimizers.Adadelta(learning_rate) elif op_type == 'adam': return optimizers.Adam(learning_rate, clipnorm=5) elif op_type == 'adamw': return AdamWeightDecay( learning_rate=learning_rate, weight_decay_rate=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-6, exclude_from_weight_decay=["layer_norm", "bias"]) elif op_type == 'adamw_2': return create_optimizer(init_lr=learning_rate, num_train_steps=9000, num_warmup_steps=0) elif op_type == 'adamw_3': return create_optimizer(init_lr=learning_rate, num_train_steps=9000, num_warmup_steps=100) else: raise ValueError('Optimizer Not Understood: {}'.format(op_type))
def descent_optimizers(): optimizers.Adadelta(learning_rate=1e-3, rho=0.95, epsilon=1e-07, name='Adadelta') optimizers.Adagrad(learning_rate=1e-3, initial_accumulator_value=0.1, epsilon=1e-07, name='Adagrad') optimizers.Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam') optimizers.Adamax(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax') optimizers.Nadam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam') optimizers.RMSprop(learning_rate=1e-3, rho=0.9, momentum=0.0, epsilon=1e-07, centered=False, name='RMSprop') optimizers.SGD(learning_rate=1e-2, momentum=0.0, nesterov=False, name='SGD')
def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float, learning_rate_decay_steps: int, ) -> tf_optimizers.Optimizer: # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop(learning_rate=learning_rate_schedule) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def compile(keras_model, loss_names=[]): """编译模型,添加损失函数,L2正则化""" # 优化器 # optimizer = optimizers.SGD(INIT_LEARNING_RATE, momentum=SGD_LEARNING_MOMENTUM, clipnorm=SGD_GRADIENT_CLIP_NORM) # optimizer = optimizers.RMSprop(learning_rate=INIT_LEARNING_RATE, rho=0.9) optimizer = optimizers.Adagrad(learning_rate=INIT_LEARNING_RATE) # optimizer = optimizers.Adadelta(learning_rate=1., rho=0.95) # optimizer = optimizers.Adam(learning_rate=INIT_LEARNING_RATE, beta_1=0.9, beta_2=0.999, amsgrad=False) # 添加损失函数,首先清除损失,防止重复计算 # keras_model._losses = [] # keras_model._per_input_losses = {} for loss_name in loss_names: loss_layer = keras_model.get_layer(loss_name) if loss_layer is None: continue loss = loss_layer.output * SEGMENT_LOSS_WEIGHTS.get(loss_name, 1.) keras_model.add_loss(loss) # 添加L2正则化,跳过Batch Normalization的gamma和beta权重 reg_losses = [ regularizers.l2(L2_WEIGHT_DECAY)(w) / tf.cast(tf.size(w), tf.float32) for w in keras_model.trainable_weights if "gamma" not in w.name and "beta" not in w.name ] keras_model.add_loss(lambda: tf.reduce_sum(reg_losses)) # 编译, 使用虚拟损失 keras_model.compile(optimizer=optimizer, loss=[None] * len(keras_model.outputs)) # 为每个损失函数增加度量 add_metrics(keras_model, metric_name_list=loss_names)
def main(): #file = r'./db/fucDatasetReg_1F_NoLinear.csv' #file = r'./db/fucDatasetReg_2F.csv' file = r'./db/fucDatasetReg_3F_1000.csv' x_train, x_test, y_train, y_test = getCsvDataset(file) lr = 1e-3 EPOCHES = 200 # optimizer = optimizerTf(lr=lr) # losses,_ = trainModel(x_train,y_train,optimizer,epochs=EPOCHES) # plotLoss(losses) opts = [] # fast group opts.append((optimizers.SGD(learning_rate=lr), 'SGD')) opts.append((optimizers.RMSprop(learning_rate=lr), 'RMSprop')) opts.append((optimizers.Adam(learning_rate=lr), 'Adam')) opts.append((optimizers.Adamax(learning_rate=lr), 'Adamax')) opts.append((optimizers.Nadam(learning_rate=lr), 'Nadam')) # # slow group opts.append((optimizers.Adadelta(learning_rate=lr), 'Adadelta')) opts.append((optimizers.Adagrad(learning_rate=lr), 'Adagrad')) opts.append((optimizers.Ftrl(learning_rate=lr), 'Ftrl')) lossesDict = {} for opti, name in opts: losses, _ = trainModel(x_train, y_train, opti, epochs=EPOCHES) lossesDict[name] = losses #print(name, losses) plotLossDict(lossesDict)
def fit_generator(self, train_generator, val_generator=None, epochs=10, monitor='val_loss', patience_count=10, metrics=['accuracy'], outdir=""): self.model.compile(optimizer=optimizers.Adagrad(), loss='categorical_crossentropy', metrics=metrics) training_callbacks = [ callbacks.EarlyStopping(monitor=monitor, patience=patience_count), # callbacks.ModelCheckpoint(filepath=outdir + 'model.{epoch}.h5', # save_best_only=True, # monitor=monitor, # mode='auto') ] if val_generator is None: self.training_history = self.model.fit(train_generator, epochs=epochs, verbose=2) else: self.training_history = self.model.fit( train_generator, validation_data=val_generator, callbacks=training_callbacks, epochs=epochs, verbose=2)
def predict(): json_file = open('model50epochbaru.json', 'r') loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) # load weights into new model loaded_model.load_weights("model50epochbaru.h5") print("Loaded model from disk") opt2 = optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) hehe="error" # evaluate loaded model on test data loaded_model.compile(optimizer = opt2, loss = 'binary_crossentropy', metrics = ['accuracy']) haha=[] count_pork = 0 count_beef = 0 test_image = image.load_img('uploads/daging.jpeg', target_size = (128, 128)) test_image = image.img_to_array(test_image) test_image = np.expand_dims(test_image, axis = 0) result = loaded_model.predict(test_image) if result[0][0] == 0: prediction = 'pork' count_pork = count_pork + 1 else: prediction = 'beef' count_beef = count_beef + 1 print(haha) print("count_beef:" + str(count_beef)) print("count_pork:" + str(count_pork)) if count_beef != 0: hehe="beef" elif count_pork != 0: hehe="pork" return hehe
def generate(self): print(trainx.shape[1]) self.model.add(Embedding(20, 128, input_length = trainx.shape[1])) for i in range(0,self.numLayers-1): self.model.add(LSTM(self.numCells, dropout = self.drop, recurrent_dropout=self.recurrDrop, return_sequences = True, unroll = True,recurrent_activation=self.recurrActivation,bias_initializer='RandomNormal',implementation=1)) self.model.add(LSTM(self.numCells, dropout = self.drop, recurrent_dropout=self.recurrDrop, unroll = True,recurrent_activation=self.recurrActivation,bias_initializer='RandomNormal',implementation=1)) self.model.add(Dense(2,activation='softmax', bias_initializer='RandomNormal')) if(self.optim == 'adam'): optimizerx = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'adam' and self.amsgrad == 'True'): optimizerx = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'SGD'): optimizerx = optimizers.SGD(lr=0.01, momentum=0.0, nesterov=False) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'adagrad'): optimizerx = optimizers.Adagrad(lr=0.01) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'RMSprop'): optimizerx = optimizers.RMSprop(lr=0.001, rho=0.9) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'Adamax'): optimizerx = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'Nadam'): optimizerx = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy'])
def build_optimizer(type, lr, kerasDefaults): """ Set the optimizer to the appropriate Keras optimizer function based on the input string and learning rate. Other required values are set to the Keras default values Parameters ---------- type : string String to choose the optimizer Options recognized: 'sgd', 'rmsprop', 'adagrad', adadelta', 'adam' See the Keras documentation for a full description of the options lr : float Learning rate kerasDefaults : list List of default parameter values to ensure consistency between frameworks Returns ---------- The appropriate Keras optimizer function """ if type == 'sgd': return optimizers.SGD(lr=lr, decay=kerasDefaults['decay_lr'], momentum=kerasDefaults['momentum_sgd'], nesterov=kerasDefaults['nesterov_sgd']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'rmsprop': return optimizers.RMSprop(lr=lr, rho=kerasDefaults['rho'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adagrad': return optimizers.Adagrad(lr=lr, epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adadelta': return optimizers.Adadelta(lr=lr, rho=kerasDefaults['rho'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # , # clipnorm=kerasDefaults['clipnorm'], # clipvalue=kerasDefaults['clipvalue']) elif type == 'adam': return optimizers.Adam(lr=lr, beta_1=kerasDefaults['beta_1'], beta_2=kerasDefaults['beta_2'], epsilon=kerasDefaults['epsilon'], decay=kerasDefaults['decay_lr']) # ,
def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float = 1.0, learning_rate_decay_steps: int = 1000000, gradient_clip_value: float = 1000000, ) -> tf_optimizers.Optimizer: """ This function defines the optimizer used by ml4ir. Users have the option to define an ExponentialDecay learning rate schedule Arguments: optimizer_key: string optimizer name to be used as defined under ml4ir.base.config.keys.OptimizerKey learning_rate: floating point learning rate for the optimizer learning_rate_decay: floating point rate at which the learning rate will be decayed every learning_rate_decay_steps learning_rate_decay_steps: int representing number of iterations after which learning rate will be decreased exponentially gradient_clip_value: float value representing the clipvalue for gradient updates. Not setting this to a reasonable value based on the model will lead to gradient explosion and NaN losses. References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay FIXME: Define all arguments overriding tensorflow defaults in a separate file for visibility with ml4ir users """ # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def GetOpti(Optimizer,LearnRate): if(Optimizer == 'SGD'): Opti = optimizers.SGD(lr=LearnRate, momentum=0.0, nesterov=False) elif(Optimizer == 'Rmsprop'): Opti = optimizers.RMSprop(lr=LearnRate, rho=0.9) elif(Optimizer == 'Adagrad'): Opti = optimizers.Adagrad(lr=LearnRate) elif(Optimizer == 'Adam'): Opti = optimizers.Adam(lr=LearnRate, beta_1=0.9, beta_2=0.999, amsgrad=False) return Opti
def choose_optimizer(model_config, learning_rate_schedule): """ Define the optimizer used for training the RelevanceModel Users have the option to define an ExponentialDecay learning rate schedule Parameters ---------- model_config : dict model configuration doctionary Returns ------- tensorflow optimizer Notes ----- References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay https://arxiv.org/pdf/1506.01186.pdf """ if 'optimizer' not in model_config: return tf_optimizers.Adam(learning_rate=learning_rate_schedule, clipvalue=5.0) else: optimizer_key = model_config['optimizer']['key'] gradient_clip_value = model_config['optimizer']['gradient_clip_value'] if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) else: raise ValueError("Unsupported Optimizer: " + optimizer_key)
def _get_optimizer(optimizer, lr_mult=1.0): "Get optimizer with correct learning rate." if optimizer == "sgd": return optimizers.SGD(lr=0.01*lr_mult) elif optimizer == "rmsprop": return optimizers.RMSprop(lr=0.001*lr_mult) elif optimizer == "adagrad": return optimizers.Adagrad(lr=0.01*lr_mult) elif optimizer == "adam": return optimizers.Adam(lr=0.001*lr_mult) elif optimizer == "nadam": return optimizers.Nadam(lr=0.002*lr_mult) raise NotImplementedError
def callOptimizer(opt='rmsprop'): '''Function returns the optimizer to use in .fit() options: adam, sgd, rmsprop, ada_grad,ada_delta,ada_max ''' opt_dict = {'adam': optimizers.Adam(), 'sgd' : optimizers.SGD(), 'rmsprop' : optimizers.RMSprop(), 'ada_grad' : optimizers.Adagrad(), 'ada_delta': optimizers.Adadelta(), 'ada_max' : optimizers.Adamax()} return opt_dict[opt]
def definePacmanTestModel1(conf): # Define Model inputShape = (conf.input_y_dim, conf.input_x_dim, conf.c_channels) state = Input(inputShape) # pre 0 in x = Conv2D(32, (3, 3), activation='relu', padding='same', name='Conv0')(state) # conv 0 x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv1')(x) # conv 1 x = MaxPooling2D((2, 2))(x) # pooling x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv2')(x) # conv 2 x = MaxPooling2D((2, 2))(x) # pooling x = Conv2D(8, (3, 3), activation='relu', padding='valid', name='Conv3')(x) # conv 3 x = MaxPooling2D((2, 2))(x) # pooling x = Flatten()(x) # flatten x = Dense(100, activation='relu')(x) # fc x = Dense(100, activation='relu')(x) # fc qsa = Dense(conf.num_actions, activation='linear')(x) # out # Make Model model = Model(state, qsa) # Configure Optimizer if conf.optimizer == 'adadelta': optimizer = optimizers.Adadelta(lr=conf.learning_rate, decay=0.0, rho=0.95) elif conf.optimizer == 'sgd': optimizer = optimizers.SGD(lr=conf.learning_rate) elif conf.optimizer == 'adam': optimizer = optimizers.Adam(lr=conf.learning_rate) elif conf.optimizer == 'adagrad': optimizer = optimizers.Adagrad(lr=conf.learning_rate) else: print("Optimizer '{0}' not found.".format(conf.optimizer)) exit(0) return model, optimizer
def get_optimizer(optimizer_key: str, learning_rate: float, learning_rate_decay: float) -> tf_optimizers.Optimizer: if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop(learning_rate=learning_rate) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def create_optimizer(opt, learning_rate, momentum=0.9, decay=0.0, nesterov=False): """ Create optimizer operation :param opt: A string which can be one of 'sgd', 'momentum' or 'adam' :param learning_rate: A float value :param momentum: A float value :return: An optimizer operation """ assert opt in [ 'sgd', 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax', 'nadam' ] if opt == 'sgd': optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum, decay=decay, nesterov=nesterov) elif opt == 'rmsprop': optimizer = optimizers.RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-06) elif opt == 'adagrad': optimizer = optimizers.Adagrad(lr=learning_rate, epsilon=1e-06) elif opt == 'adadelta': optimizer = optimizers.Adadelta(lr=learning_rate, rho=0.95, epsilon=1e-06) elif opt == 'adam': optimizer = optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'adamax': optimizer = optimizers.Adamax(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'nadam': optimizer = optimizers.Nadam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) else: optimizer = None return optimizer
def __get_optimizer(optimizer, lr): if optimizer == 'sgd': return optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': return optimizers.RMSprop(lr=lr) elif optimizer == 'adagrad': return optimizers.Adagrad(lr=lr) elif optimizer == 'adadelta': return optimizers.Adadelta(lr=lr) elif optimizer == 'adam': return optimizers.Adam(lr=lr) elif optimizer == 'adamax': return optimizers.Adamax(lr=lr) elif optimizer == 'nadam': return optimizers.Nadam(lr=lr)
def _compile(self, model, loss_function, optimizer, lr=0.01, decay=0.0, clipnorm=0.0): """Compiles a model specified with Keras. See https://keras.io/optimizers/ for more info on each optimizer. Args: model: Keras model object to compile loss_function: Keras loss_function object to compile model with optimizer (str): the optimizer to use during training lr (float): learning rate to use during training decay (float): per epoch decay rate clipnorm (float): gradient normalization threshold """ # The parameters of these optimizers can be freely tuned. if optimizer == 'sgd': optimizer_ = optimizers.SGD(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adam': optimizer_ = optimizers.Adam(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adamax': optimizer_ = optimizers.Adamax(lr=lr, decay=decay, clipnorm=clipnorm) # It is recommended to leave the parameters of this optimizer at their # default values (except the learning rate, which can be freely tuned). # This optimizer is usually a good choice for recurrent neural networks elif optimizer == 'rmsprop': optimizer_ = optimizers.RMSprop(lr=lr, clipnorm=clipnorm) # It is recommended to leave the parameters of these optimizers at their # default values. elif optimizer == 'adagrad': optimizer_ = optimizers.Adagrad(clipnorm=clipnorm) elif optimizer == 'adadelta': optimizer_ = optimizers.Adadelta(clipnorm=clipnorm) elif optimizer == 'nadam': optimizer_ = optimizers.Nadam(clipnorm=clipnorm) else: err_msg = "Argument for `optimizer` is invalid, got: {}".format( optimizer) LOGGER.error('ValueError %s', err_msg) raise ValueError(err_msg) model.compile(optimizer=optimizer_, loss=loss_function)
def get_optimizer(args): clipvalue = 0 clipnorm = 10 if args.algorithm == 'rmsprop': optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'sgd': optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adagrad': optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adadelta': optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adam': optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) elif args.algorithm == 'adamax': optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) else: raise Exception("Can't find optimizer " + args.algorithm) return optimizer
def get_optimizer(opt_name, learning_rate=None): if opt_name == 'sgd': optimizer = optimizers.SGD( 0.01 if learning_rate is None else learning_rate, momentum=0.9, nesterov=True) elif opt_name == 'rmsprop': optimizer = optimizers.RMSprop( 0.001 if learning_rate is None else learning_rate) elif opt_name == 'adagrad': optimizer = optimizers.Adagrad( 0.01 if learning_rate is None else learning_rate) elif opt_name == 'adam': optimizer = optimizers.Adam( 0.001 if learning_rate is None else learning_rate, amsgrad=True) return optimizer
def set_optimizers(self, select_optimizer, select_lr): print("optimizers setting.") print("optimizers : ", select_optimizer) # Configure the optimizer from config. if select_optimizer == "adam" or select_optimizer == "Adam": opt = optimizers.Adam(lr=select_lr) elif select_optimizer == "sgd" or select_optimizer == "SGD": opt = optimizers.SGD(lr=select_lr) elif select_optimizer == "adagrad" or select_optimizer == "Adagrad": opt = optimizers.Adagrad(lr=select_lr) elif select_optimizer == "adadelta" or select_optimizer == "Adadelta": opt = optimizers.Adadelta(lr=select_lr) else: print("This is an unconfigured optimizer that uses Adam instead.") opt = optimizers.Adam(lr=select_lr) print("optimizer setting ... ok.") return opt
def get_model(self, number_of_dense_layer=4, number_of_ff_node=640, optimizer='sgd', learning_rate=0.01): input_sequence = layers.Input(shape=(self.audio_sample_length, self.M)) conv_layer = layers.Conv1D(self.F, self.impulse_reponse_length, padding='same')(input_sequence) hop_and_pooling_layer = layers.Lambda(self.hop_and_maxpooling, output_shape=self.get_shape_hop_and_maxpooling, trainable=False)(conv_layer) full_connect_layer = layers.Dense(number_of_ff_node, activation='relu')(hop_and_pooling_layer) for _ in range(0, number_of_dense_layer): full_connect_layer = layers.Dense(number_of_ff_node, activation='relu')(full_connect_layer) full_connect_layer = layers.Dense(number_of_ff_node, activation='relu')(full_connect_layer) full_connect_layer = layers.Dense(number_of_ff_node, activation='relu')(full_connect_layer) full_connect_layer = layers.Dense(self.number_of_class, activation='softmax')(full_connect_layer) model = Model(inputs=input_sequence, outputs=full_connect_layer) if optimizer == 'sgd': model.compile(optimizer=optimizers.SGD(lr=learning_rate), loss=losses.categorical_crossentropy, metrics=['accuracy']) elif optimizer == 'adagrad': model.compile(optimizer=optimizers.Adagrad(lr=learning_rate), loss=losses.categorical_crossentropy, metrics=['accuracy']) return model
def cc_optimizer(self, learning_rate, decay_rate=0, optimizer='adam'): if optimizer == 'sgd': self.cc_optimizer = optimizers.SGD(lr=learning_rate,\ decay = decay_rate, \ momentum = moment, \ nesterov=True) elif optimizer == 'rms': #-------------------------------------------------------------- self.cc_optimizer = optimizers.RMSprop(lr = learning_rate, \ rho= 0.9, \ epsilon = None,\ decay = decay_rate) elif optimizer == 'adagrad': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adagrad (lr = learning_rate , \ epsilon = None , \ decay = decay_rate) elif optimizer == 'adadelta': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adadelta(lr = learning_rate, \ rho=0.95 , \ epsilon = None,\ decay = decay_rate) elif optimizer == 'nadam': self.cc_optimizer = optimizers.Nadam(lr = learning_rate, \ beta_1 = 0.9, \ beta_2 = 0.999, \ epsilon = None, \ schedule_decay = 0.004) else: self.cc_optimizer = optimizers.Adam(lr = learning_rate, \ beta_1 = 0.9 , \ beta_2 = 0.999 , \ epsilon = None,\ decay = decay_rate,\ amsgrad = True ) return self.cc_optimizer
def train(num_epochs, start_epoch=0, model_type="horizontal", model_struc="resnet_lstm"): backend.set_learning_phase(True) crnn = CRNN(model_type=model_type, model_struc=model_struc) model = crnn.model_for_training() model.compile(optimizer=optimizers.Adagrad(learning_rate=0.01), loss={ "ctc_loss": lambda y_true, out_loss: out_loss }) if start_epoch > 0: weights_prefix = os.path.join( CRNN_CKPT_DIR, model_type + "_" + model_struc + "_crnn_weights_%05d_" % start_epoch) model.load_weights(filepath=weights_prefix) check_or_makedirs(CRNN_CKPT_DIR) ckpt_path = os.path.join( CRNN_CKPT_DIR, model_type + "_" + model_struc + "_crnn_weights_{epoch:05d}_{val_loss:.2f}.tf") checkpoint = callbacks.ModelCheckpoint(filepath=ckpt_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode="min") model.fit_generator( generator=create_text_lines_batch(type=model_type, batch_size=BATCH_SIZE_TEXT_LINE), steps_per_epoch=100, epochs=start_epoch + num_epochs, verbose=1, callbacks=[checkpoint], validation_data=load_text_lines_batch(type=model_type, batch_size=BATCH_SIZE_TEXT_LINE), validation_steps=50, max_queue_size=50, workers=2, use_multiprocessing=True, initial_epoch=start_epoch)
def make_optimizer(name: str, lr: Optional[float], clipnorm: float) -> optimizers.Optimizer: if name == 'sgd': lr = lr or 0.01 return optimizers.SGD(lr=lr, clipnorm=clipnorm) elif name == 'adagrad': lr = lr or 0.01 return optimizers.Adagrad(lr=lr, clipnorm=clipnorm) elif name == 'adam': lr = lr or 0.001 return optimizers.Adam(lr=lr, clipnorm=clipnorm) elif name == 'adamax': lr = lr or 0.001 return optimizers.Adamax(lr=lr, clipnorm=clipnorm) elif name == 'nadam': lr = lr or 0.001 return optimizers.Nadam(lr=lr, clipnorm=clipnorm) else: raise NotImplementedError
def setOptimizer(self, config): configOptimizer = config["model"]["optimizer"].lower() if configOptimizer == "Adadelta".lower(): self.optimizer = optimizers.Adadelta() elif configOptimizer == "Adagrad".lower(): self.optimizer = optimizers.Adagrad() elif configOptimizer == "Adamax".lower(): self.optimizer = optimizers.Adamax() elif configOptimizer == "Ftrl".lower(): self.optimizer = optimizers.Ftrl() elif configOptimizer == "SGD".lower(): self.optimizer = optimizers.SGD() elif configOptimizer == "Nadam".lower(): self.optimizer = optimizers.Nadam() elif configOptimizer == "Optimizer".lower(): self.optimizer = optimizers.Optimizer() elif configOptimizer == "RMSprop".lower(): self.optimizer = optimizers.RMSprop()
def set_optimizer(self, optimizer, loss="categorical_crossentropy"): if optimizer.lower() == "adam": opt_handle = Opt.Adam() elif optimizer.lower() == "adagrad": opt_handle = Opt.Adagrad() elif optimizer.lower() == "adadelta": opt_handle = Opt.Adadelta() elif optimizer.lower() == "rmsprop": opt_handle = Opt.RMSprop() else: print("Unknown optimizer {}. Using Adam!".format(optimizer)) opt_handle = Opt.Adam() print("Setting model optimizer to {}".format(optimizer)) self.model.compile( loss="categorical_crossentropy", optimizer=opt_handle, metrics=["accuracy"], )
def get_optimizer(): optimizer_name = optimizer_names[random.randint(0, len(optimizer_names) - 1)] model_attributes.optimizer_name = optimizer_name if optimizer_name == 'SGD': return optimizers.SGD(lr=get_learning_rate()) elif optimizer_name == 'RMSprop': return optimizers.RMSprop(lr=get_learning_rate()) elif optimizer_name == 'Adagrad': return optimizers.Adagrad(lr=get_learning_rate()) elif optimizer_name == 'Adadelta': return optimizers.Adadelta(lr=get_learning_rate()) elif optimizer_name == 'Adam': return optimizers.Adam(lr=get_learning_rate()) elif optimizer_name == 'Adamax': return optimizers.Adamax(lr=get_learning_rate()) elif optimizer_name == 'Nadam': return optimizers.Nadam(lr=get_learning_rate()) return None
def _load_architecture(self, config): ''' load the architecture configs ''' # define default network configuration self.architecture = { "layers": [200], "loss_function": "categorical_crossentropy", "Dropout": 0.2, "L1_Norm": 0., "L2_Norm": 1e-5, "batch_size": 5000, "optimizer": optimizers.Adagrad(decay=0.99), "activation_function": "elu", "output_activation": "Softmax", "earlystopping_percentage": None, "earlystopping_epochs": None, } for key in config: self.architecture[key] = config[key]