def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float, learning_rate_decay_steps: int, ) -> tf_optimizers.Optimizer: # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop(learning_rate=learning_rate_schedule) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def descent_optimizers(): optimizers.Adadelta(learning_rate=1e-3, rho=0.95, epsilon=1e-07, name='Adadelta') optimizers.Adagrad(learning_rate=1e-3, initial_accumulator_value=0.1, epsilon=1e-07, name='Adagrad') optimizers.Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam') optimizers.Adamax(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax') optimizers.Nadam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam') optimizers.RMSprop(learning_rate=1e-3, rho=0.9, momentum=0.0, epsilon=1e-07, centered=False, name='RMSprop') optimizers.SGD(learning_rate=1e-2, momentum=0.0, nesterov=False, name='SGD')
def SLFNN(vst_onlyTokens, dl_terms, dl_associations, vso, nbEpochs=100, batchSize=64): vstTerm, l_unknownToken = word2term.wordVST2TermVST( vst_onlyTokens, dl_terms) data, labels = getMatrix(dl_terms, vstTerm, dl_associations, vso, symbol="___") inputSize = data.shape[1] ontoSpaceSize = labels.shape[1] model = models.Sequential() model.add( layers.Dense(units=ontoSpaceSize, use_bias=True, kernel_initializer=initializers.GlorotUniform(), input_shape=(inputSize, ))) model.summary() model.compile( optimizer=optimizers.Nadam(), loss=losses.LogCosh(), metrics=[metrics.CosineSimilarity(), metrics.MeanSquaredError()]) model.fit(data, labels, epochs=nbEpochs, batch_size=batchSize) return model, vso, l_unknownToken
def generate(self): print(trainx.shape[1]) self.model.add(Embedding(20, 128, input_length = trainx.shape[1])) for i in range(0,self.numLayers-1): self.model.add(LSTM(self.numCells, dropout = self.drop, recurrent_dropout=self.recurrDrop, return_sequences = True, unroll = True,recurrent_activation=self.recurrActivation,bias_initializer='RandomNormal',implementation=1)) self.model.add(LSTM(self.numCells, dropout = self.drop, recurrent_dropout=self.recurrDrop, unroll = True,recurrent_activation=self.recurrActivation,bias_initializer='RandomNormal',implementation=1)) self.model.add(Dense(2,activation='softmax', bias_initializer='RandomNormal')) if(self.optim == 'adam'): optimizerx = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'adam' and self.amsgrad == 'True'): optimizerx = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'SGD'): optimizerx = optimizers.SGD(lr=0.01, momentum=0.0, nesterov=False) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'adagrad'): optimizerx = optimizers.Adagrad(lr=0.01) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'RMSprop'): optimizerx = optimizers.RMSprop(lr=0.001, rho=0.9) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'Adamax'): optimizerx = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy']) if(self.optim == 'Nadam'): optimizerx = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999) self.model.compile(loss = self.loss, optimizer = optimizerx,metrics=['accuracy'])
def metrics(loader, size = 128): # import tensorflow.keras.backend as K # # def L1(y_true, y_pred): # return K.sqrt(K.mean(K.square(y_pred - y_true))) model = tf.keras.models.load_model("log/20191130-003024/model-snap70000.h5") adam = optimizers.Nadam() model.compile(adam, 'logcosh', metrics=[tf.keras.metrics.RootMeanSquaredError(), tf.keras.metrics.MeanAbsoluteError()]) length = 10000 LOSS = np.zeros(shape=(length)) RMSE = np.zeros(shape=(length)) MAE = np.zeros(shape=(length)) for i in range(length): x, y = loader.get_validation_batch(size) loss, rmse, mae = model.test_on_batch(x, y) LOSS[i] = np.mean(loss) RMSE[i] = np.mean(rmse) MAE[i] = np.mean(mae) print_log(f"running {i}, LOSS = {LOSS[i]}, RMSE = {RMSE[i]}, MAE = {MAE[i]}") LOSS = np.mean(LOSS) RMSE = np.mean(RMSE) MAE = np.mean(MAE) print_log(f"LOSS = {LOSS}, RMSE = {RMSE}, MAE = {MAE}") return LOSS, RMSE, MAE
def get_model(character_label_encoder, dropout=0.05, n_units=64): n_character_classes = len(character_label_encoder.classes_) # Note: input shape None-by-n_character_classes allows for arbitrary length sentences input_layer = Input(shape=(None, n_character_classes)) gru1 = GRU(units=n_units, return_sequences=True, dropout=dropout,)(input_layer) gru2 = GRU(units=n_units, return_sequences=True, dropout=dropout,)(gru1) # Note: activation=None means linear activation (used for regression output) gru_result = GRU(units=1, activation=None, name=OUTPUT_RESULT)(gru2) gru_first_number = GRU(units=1, activation=None, name=OUTPUT_FIRST_NUMBER)(gru2) model = Model(inputs=input_layer, outputs=[gru_result, gru_first_number]) nadam = optimizers.Nadam() model.compile( optimizer=nadam, loss=losses.mean_squared_error, metrics=["mean_squared_error"], ) print(model.summary()) return model
def main(): #file = r'./db/fucDatasetReg_1F_NoLinear.csv' #file = r'./db/fucDatasetReg_2F.csv' file = r'./db/fucDatasetReg_3F_1000.csv' x_train, x_test, y_train, y_test = getCsvDataset(file) lr = 1e-3 EPOCHES = 200 # optimizer = optimizerTf(lr=lr) # losses,_ = trainModel(x_train,y_train,optimizer,epochs=EPOCHES) # plotLoss(losses) opts = [] # fast group opts.append((optimizers.SGD(learning_rate=lr), 'SGD')) opts.append((optimizers.RMSprop(learning_rate=lr), 'RMSprop')) opts.append((optimizers.Adam(learning_rate=lr), 'Adam')) opts.append((optimizers.Adamax(learning_rate=lr), 'Adamax')) opts.append((optimizers.Nadam(learning_rate=lr), 'Nadam')) # # slow group opts.append((optimizers.Adadelta(learning_rate=lr), 'Adadelta')) opts.append((optimizers.Adagrad(learning_rate=lr), 'Adagrad')) opts.append((optimizers.Ftrl(learning_rate=lr), 'Ftrl')) lossesDict = {} for opti, name in opts: losses, _ = trainModel(x_train, y_train, opti, epochs=EPOCHES) lossesDict[name] = losses #print(name, losses) plotLossDict(lossesDict)
def training_VALModel(opt_type='sgd',steps=100,epochs=10,batch_size=1,depth='v1'): K.clear_session() model = get_val_model(depth) opt = None if opt_type == 'rms': #opt= optimizers.RMSprop(lr=1e-3, rho=0.9, epsilon=1e-8) opt= optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004) else: opt = optimizers.SGD(lr=1e-4) if batch_size > 1: model.compile(loss=losses.binary_crossentropy, optimizer=opt, metrics = ['acc']) else: model.compile(loss=losses.binary_crossentropy, optimizer=opt, metrics = ['acc']) try: model.load_weights("{}-weights-{}.h5".format('VALModel',depth), by_name=True) pass except: print('not load') pass filepath="{}-weights-{}.h5".format('VALModel',depth) es = EarlyStopping(monitor='val_loss', mode='min', verbose=0,patience=5) ck = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min') history = model.fit_generator(data_gen('train',batch_size),steps_per_epoch=steps , epochs=epochs, validation_data=data_gen('val',4), validation_steps=100,callbacks=[ck,es],verbose=1) with open('visual_{}.pkl'.format(depth),'wb') as fp: pickle.dump(history.history, fp) del model
def compile_model(model): model.compile(optimizer=optimizers.Nadam(), loss=losses.SparseCategoricalCrossentropy(), metrics=[ metrics.SparseCategoricalAccuracy(), metrics.SparseTopKCategoricalAccuracy(5) ]) return model
def SCNN(vst_onlyTokens, dl_terms, dl_associations, vso, nbEpochs=150, batchSize=64, l_numberOfFilters=[4000], l_filterSizes=[1], phraseMaxSize=15): data, labels, l_unkownTokens, l_uncompleteExpressions = prepare2D_data( vst_onlyTokens, dl_terms, dl_associations, vso, phraseMaxSize) embeddingSize = data.shape[2] ontoSpaceSize = labels.shape[2] inputLayer = Input(shape=(phraseMaxSize, embeddingSize)) l_subLayers = list() for i, filterSize in enumerate(l_filterSizes): convLayer = (layers.Conv1D( l_numberOfFilters[i], filterSize, strides=1, kernel_initializer=initializers.GlorotUniform()))(inputLayer) outputSize = phraseMaxSize - filterSize + 1 pool = (layers.MaxPool1D(pool_size=outputSize))(convLayer) activationLayer = (layers.LeakyReLU(alpha=0.3))(pool) l_subLayers.append(activationLayer) if len(l_filterSizes) > 1: concatenateLayer = (layers.Concatenate(axis=-1))( l_subLayers) # axis=-1 // concatenating on the last dimension else: concatenateLayer = l_subLayers[0] convModel = Model(inputs=inputLayer, outputs=concatenateLayer) fullmodel = models.Sequential() fullmodel.add(convModel) fullmodel.add( layers.Dense(ontoSpaceSize, kernel_initializer=initializers.GlorotUniform())) fullmodel.summary() fullmodel.compile( optimizer=optimizers.Nadam(), loss=losses.LogCosh(), metrics=[metrics.CosineSimilarity(), metrics.MeanSquaredError()]) fullmodel.fit(data, labels, epochs=nbEpochs, batch_size=batchSize) return fullmodel, vso, l_unkownTokens
def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float = 1.0, learning_rate_decay_steps: int = 1000000, gradient_clip_value: float = 1000000, ) -> tf_optimizers.Optimizer: """ This function defines the optimizer used by ml4ir. Users have the option to define an ExponentialDecay learning rate schedule Arguments: optimizer_key: string optimizer name to be used as defined under ml4ir.base.config.keys.OptimizerKey learning_rate: floating point learning rate for the optimizer learning_rate_decay: floating point rate at which the learning rate will be decayed every learning_rate_decay_steps learning_rate_decay_steps: int representing number of iterations after which learning rate will be decreased exponentially gradient_clip_value: float value representing the clipvalue for gradient updates. Not setting this to a reasonable value based on the model will lead to gradient explosion and NaN losses. References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay FIXME: Define all arguments overriding tensorflow defaults in a separate file for visibility with ml4ir users """ # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def train(): model = xmodel.getModel() generator = preprocessor.getLoader(batchsize=BATCH_SIZE) valgenerator = preprocessor.getLoader(mode='validation',batchsize=BATCH_SIZE) #sgd = optimizers.SGD(learning_rate=0.05, momentum=0.01, nesterov=True) nadam = optimizers.Nadam()# apparently according to keras, the default is the recommended value model.compile(optimizer='nadam', loss="mse", metrics = ['mae']) save_callback = callbacks.ModelCheckpoint('ckpts/weights.{epoch:d}-{loss:.2f}-{val_loss:.2f}-{val_acc:.2f}.hdf5')# add validation tb = callbacks.TensorBoard(log_dir='./logs') model.fit_generator(generator, steps_per_epoch=int(36450/BATCH_SIZE), epochs=100, verbose=1, validation_freq=1, validation_data=valgenerator,validation_steps=int(4050/BATCH_SIZE), callbacks=[save_callback,tb])
def choose_optimizer(model_config, learning_rate_schedule): """ Define the optimizer used for training the RelevanceModel Users have the option to define an ExponentialDecay learning rate schedule Parameters ---------- model_config : dict model configuration doctionary Returns ------- tensorflow optimizer Notes ----- References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay https://arxiv.org/pdf/1506.01186.pdf """ if 'optimizer' not in model_config: return tf_optimizers.Adam(learning_rate=learning_rate_schedule, clipvalue=5.0) else: optimizer_key = model_config['optimizer']['key'] gradient_clip_value = model_config['optimizer']['gradient_clip_value'] if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value if 'gradient_clip_value' in model_config['optimizer'] else 5.0) else: raise ValueError("Unsupported Optimizer: " + optimizer_key)
def train_from_ckpt(img_dir, annotation_json_fp, segmentation_dir, num_sectors, vehicle_only, ckpt_fp): num_sectors = int(num_sectors) backbone = load_model(ckpt_fp, custom_objects={'quality_loss': quality_loss}) backbone.layers.pop() predictions = Dense(num_sectors, activation='softmax', name="final_layer")(backbone.layers[-1].output) model = Model(backbone.input, outputs=predictions) if vehicle_only.lower() in [ 'true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'uh-huh' ]: generator = preprocessor.getLoader(image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE, discard_cls=[0, 1, 2, 4, 7]) valgenerator = preprocessor.getLoader(mode='validation', image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE, discard_cls=[0, 1, 2, 4, 7]) else: generator = preprocessor.getLoader(image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE) valgenerator = preprocessor.getLoader(mode='validation', image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE) #sgd = optimizers.SGD(learning_rate=0.05, momentum=0.01, nesterov=True) nadam = optimizers.Nadam() model.compile(optimizer='nadam', loss=quality_loss) save_callback = callbacks.ModelCheckpoint( 'ckpts/weights.{epoch:d}-{loss:.2f}-{val_loss:.2f}-{val_acc:.2f}.hdf5' ) # add validation tb = callbacks.TensorBoard(log_dir='./logs') model.fit_generator(generator, steps_per_epoch=int(36450 / BATCH_SIZE), epochs=100, verbose=1, validation_freq=1, validation_data=valgenerator, validation_steps=int(4050 / BATCH_SIZE), callbacks=[save_callback, tb])
def _get_optimizer(optimizer, lr_mult=1.0): "Get optimizer with correct learning rate." if optimizer == "sgd": return optimizers.SGD(lr=0.01*lr_mult) elif optimizer == "rmsprop": return optimizers.RMSprop(lr=0.001*lr_mult) elif optimizer == "adagrad": return optimizers.Adagrad(lr=0.01*lr_mult) elif optimizer == "adam": return optimizers.Adam(lr=0.001*lr_mult) elif optimizer == "nadam": return optimizers.Nadam(lr=0.002*lr_mult) raise NotImplementedError
def __init__(self, layer_size): super().__init__() self.add(layers.Dense(1000, activation='relu', input_shape=(layer_size,), name='Hidden-1')) self.add(layers.Dropout(0.1)) self.add(layers.Dense(500, activation='relu', name='Hidden-2')) self.add(layers.Dropout(0.1)) self.add(layers.Dense(100, activation='relu', name='Hidden-3')) self.add(layers.Dropout(0.1)) self.add(layers.Dense(10, activation='relu', name='Hidden-4')) self.add(layers.Dropout(0.1)) self.add(layers.Dense(1, activation='sigmoid')) optimizer = optimizers.Nadam(lr=0.0000005) self.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
def create_model(cfg, submodel_settings, mdl_data=None, ticker_name='', train_mode=True, learning_rate=0.002, input_shape=None): # print(f'model> clear backend session') K.clear_session() if mdl_data is None: num_samples = input_shape[0] num_features = input_shape[-1] input_length = submodel_settings.lookback_days else: num_samples = mdl_data.shape[0] num_features = len(mdl_data.X.head(1).tolist()[0][0][0][0]) input_length = submodel_settings.lookback_days input_dim = num_features lstm_dim = cfg.model.lstm_hidden_size output_dim = 1 mdl = Sequential() mdl.add(BatchNormalization(input_shape=(input_length, input_dim))) mdl.add(Masking()) mdl.add( LSTM(lstm_dim, dropout=.2, recurrent_dropout=.2, return_sequences=True, activation="softsign")) mdl.add( LSTM(lstm_dim, dropout=.2, recurrent_dropout=.2, return_sequences=True, activation="softsign")) mdl.add( LSTM(lstm_dim, dropout=.2, recurrent_dropout=.2, activation="softsign")) mdl.add(Dense(output_dim)) optimizer = optimizers.Nadam(learning_rate=learning_rate, beta_1=0.9, beta_2=0.999) mdl.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mean_absolute_error', 'mean_squared_error']) if train_mode: print(f'model> model created\n:{mdl.summary()}') load_weights(cfg, submodel_settings, mdl, ticker_name, train_mode) return mdl
def train_from_8_ckpt(ckpt, only_vehicles = True): backbone = load_model(ckpt) if only_vehicles: generator = preprocessor.getLoader(batchsize=BATCH_SIZE, discard_cls = [0,1,2,4,7]) valgenerator = preprocessor.getLoader(mode='validation',batchsize=BATCH_SIZE, discard_cls = [0,1,2,4,7]) else: generator = preprocessor.getLoader(batchsize=BATCH_SIZE) valgenerator = preprocessor.getLoader(mode='validation',batchsize=BATCH_SIZE) #sgd = optimizers.SGD(learning_rate=0.05, momentum=0.01, nesterov=True) nadam = optimizers.Nadam() backbone.compile(optimizer='nadam', loss="mse", metrics = ['mae']) save_callback = callbacks.ModelCheckpoint('ckpts/weights.{epoch:d}-{loss:.2f}-{val_loss:.2f}.hdf5')# add validation tb = callbacks.TensorBoard(log_dir='./logs') backbone.fit_generator(generator, steps_per_epoch=int(36450/BATCH_SIZE), epochs=100, verbose=1, validation_freq=1, validation_data=valgenerator,validation_steps=int(4050/BATCH_SIZE), callbacks=[save_callback,tb])
def get_optimizer(optimizer_key: str, learning_rate: float, learning_rate_decay: float) -> tf_optimizers.Optimizer: if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD(learning_rate=learning_rate) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop(learning_rate=learning_rate) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def create_optimizer(opt, learning_rate, momentum=0.9, decay=0.0, nesterov=False): """ Create optimizer operation :param opt: A string which can be one of 'sgd', 'momentum' or 'adam' :param learning_rate: A float value :param momentum: A float value :return: An optimizer operation """ assert opt in [ 'sgd', 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax', 'nadam' ] if opt == 'sgd': optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum, decay=decay, nesterov=nesterov) elif opt == 'rmsprop': optimizer = optimizers.RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-06) elif opt == 'adagrad': optimizer = optimizers.Adagrad(lr=learning_rate, epsilon=1e-06) elif opt == 'adadelta': optimizer = optimizers.Adadelta(lr=learning_rate, rho=0.95, epsilon=1e-06) elif opt == 'adam': optimizer = optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'adamax': optimizer = optimizers.Adamax(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) elif opt == 'nadam': optimizer = optimizers.Nadam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) else: optimizer = None return optimizer
def train(img_dir, annotation_json_fp, segmentation_dir, num_sectors, vehicle_only): num_sectors = int(num_sectors) model = xmodel.getModel(ry_cats=num_sectors) if vehicle_only.lower() in [ 'true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'uh-huh' ]: generator = preprocessor.getLoader(image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE, discard_cls=[0, 1, 2, 4, 7]) valgenerator = preprocessor.getLoader(mode='validation', image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE, discard_cls=[0, 1, 2, 4, 7]) else: generator = preprocessor.getLoader(image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE) valgenerator = preprocessor.getLoader(mode='validation', image_dir=img_dir, jsonfp=annotation_json_fp, segs_dir=segmentation_dir, ry_cats=num_sectors, batchsize=BATCH_SIZE) #sgd = optimizers.SGD(learning_rate=0.05, momentum=0.01, nesterov=True) nadam = optimizers.Nadam( ) # apparently according to keras, the default is the recommended value model.compile(optimizer='nadam', loss=quality_loss) save_callback = callbacks.ModelCheckpoint( './ckpts/weights.{epoch:d}-{loss:.2f}-{val_loss:.2f}.hdf5' ) # add validation tb = callbacks.TensorBoard(log_dir='./logs') model.fit_generator(generator, steps_per_epoch=int(36450 / BATCH_SIZE), epochs=100, verbose=1, validation_freq=1, validation_data=valgenerator, validation_steps=int(4050 / BATCH_SIZE), callbacks=[save_callback, tb])
def __get_optimizer(optimizer, lr): if optimizer == 'sgd': return optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': return optimizers.RMSprop(lr=lr) elif optimizer == 'adagrad': return optimizers.Adagrad(lr=lr) elif optimizer == 'adadelta': return optimizers.Adadelta(lr=lr) elif optimizer == 'adam': return optimizers.Adam(lr=lr) elif optimizer == 'adamax': return optimizers.Adamax(lr=lr) elif optimizer == 'nadam': return optimizers.Nadam(lr=lr)
def _compile(self, model, loss_function, optimizer, lr=0.01, decay=0.0, clipnorm=0.0): """Compiles a model specified with Keras. See https://keras.io/optimizers/ for more info on each optimizer. Args: model: Keras model object to compile loss_function: Keras loss_function object to compile model with optimizer (str): the optimizer to use during training lr (float): learning rate to use during training decay (float): per epoch decay rate clipnorm (float): gradient normalization threshold """ # The parameters of these optimizers can be freely tuned. if optimizer == 'sgd': optimizer_ = optimizers.SGD(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adam': optimizer_ = optimizers.Adam(lr=lr, decay=decay, clipnorm=clipnorm) elif optimizer == 'adamax': optimizer_ = optimizers.Adamax(lr=lr, decay=decay, clipnorm=clipnorm) # It is recommended to leave the parameters of this optimizer at their # default values (except the learning rate, which can be freely tuned). # This optimizer is usually a good choice for recurrent neural networks elif optimizer == 'rmsprop': optimizer_ = optimizers.RMSprop(lr=lr, clipnorm=clipnorm) # It is recommended to leave the parameters of these optimizers at their # default values. elif optimizer == 'adagrad': optimizer_ = optimizers.Adagrad(clipnorm=clipnorm) elif optimizer == 'adadelta': optimizer_ = optimizers.Adadelta(clipnorm=clipnorm) elif optimizer == 'nadam': optimizer_ = optimizers.Nadam(clipnorm=clipnorm) else: err_msg = "Argument for `optimizer` is invalid, got: {}".format( optimizer) LOGGER.error('ValueError %s', err_msg) raise ValueError(err_msg) model.compile(optimizer=optimizer_, loss=loss_function)
def create_model(neurons=1): model = Sequential() # Convolutional layers model.add( Conv2D(32, (3, 3), input_shape=input_shape, activation="relu", padding='same')) model.add(Dropout(0.3)) model.add(BatchNormalization()) model.add(Conv2D(64, (3, 3), activation="relu", padding='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.3)) model.add(BatchNormalization()) model.add(Flatten()) model.add(Dropout(0.3)) model.add( Dense(neurons, kernel_constraint=maxnorm(3), kernel_initializer='lecun_uniform')) model.add(Activation("relu")) model.add(Dropout(0.3)) model.add(BatchNormalization()) model.add( Dense(neurons / 2, kernel_constraint=maxnorm(3), kernel_initializer='lecun_uniform')) model.add(Activation("relu")) model.add(Dropout(0.3)) model.add(BatchNormalization()) model.add(Dense(class_num, kernel_initializer='lecun_uniform') ) #Final layer has same number of neurons as classes model.add(Activation('softmax')) optimizer = optimizers.Nadam() model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) es_callback = EarlyStopping(monitor='val_loss', patience=10) return model
def train(args): filepath = "weights-{epoch:03d}-{val_loss:.4f}-{val_mean_iou:.4f}.h5" weights_dir = os.path.join(args.weights, args.backBone + '_' + args.model) cfg.check_folder(weights_dir) model_weights = os.path.join(weights_dir, filepath) # build the model model, base_model = builder(cfg.n_classes, (256, 256), args.model, args.backBone) model.summary() # compile the model #sgd = optimizers.SGD(lr=cfg.lr, momentum=0.9) nadam = optimizers.Nadam(lr=cfg.lr, beta_1=0.9, beta_2=0.999, epsilon=1e-8) model.compile(optimizer=nadam, loss='categorical_crossentropy', metrics=[MeanIoU(cfg.n_classes)]) # checkpoint setting model_checkpoint = ModelCheckpoint(model_weights, monitor='val_loss', save_best_only=True, mode='auto') # learning rate scheduler setting lr_decay = lr_decays_func(args.lr_scheduler, args.learning_rate, args.num_epochs, args.lr_warmup) learning_rate_scheduler = LearningRateScheduler(lr_decay, args.learning_rate, args.lr_warmup, cfg.steps_per_epoch, num_epochs=args.num_epochs, verbose=1) callbacks = [model_checkpoint] # training... train_set = dataloader.train_data_generator(cfg.train_data_path, cfg.train_label_path, cfg.batch_size, cfg.n_classes, cfg.data_augment) val_set = dataloader.val_data_generator(cfg.val_data_path, cfg.val_label_path, cfg.batch_size, cfg.n_classes) start_epoch = 0 if os.path.exists(weights_dir) and os.listdir(weights_dir): a = sorted(file for file in os.listdir(weights_dir)) model.load_weights(weights_dir + '/' + a[-1], by_name=True) # if load success, output info print('loaded :' + '-' * 8 + weights_dir + '/' + a[-1]) start_epoch = int(a[-1][8:11]) model.fit(train_set, steps_per_epoch=cfg.steps_per_epoch, epochs=args.num_epochs, callbacks=callbacks, validation_data=val_set, validation_steps=cfg.validation_steps, max_queue_size= cfg.batch_size, initial_epoch=start_epoch)
def cc_optimizer(self, learning_rate, decay_rate=0, optimizer='adam'): if optimizer == 'sgd': self.cc_optimizer = optimizers.SGD(lr=learning_rate,\ decay = decay_rate, \ momentum = moment, \ nesterov=True) elif optimizer == 'rms': #-------------------------------------------------------------- self.cc_optimizer = optimizers.RMSprop(lr = learning_rate, \ rho= 0.9, \ epsilon = None,\ decay = decay_rate) elif optimizer == 'adagrad': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adagrad (lr = learning_rate , \ epsilon = None , \ decay = decay_rate) elif optimizer == 'adadelta': #-------------------------------------------------------------- self.cc_optimizer = optimizers.Adadelta(lr = learning_rate, \ rho=0.95 , \ epsilon = None,\ decay = decay_rate) elif optimizer == 'nadam': self.cc_optimizer = optimizers.Nadam(lr = learning_rate, \ beta_1 = 0.9, \ beta_2 = 0.999, \ epsilon = None, \ schedule_decay = 0.004) else: self.cc_optimizer = optimizers.Adam(lr = learning_rate, \ beta_1 = 0.9 , \ beta_2 = 0.999 , \ epsilon = None,\ decay = decay_rate,\ amsgrad = True ) return self.cc_optimizer
def make_optimizer(name: str, lr: Optional[float], clipnorm: float) -> optimizers.Optimizer: if name == 'sgd': lr = lr or 0.01 return optimizers.SGD(lr=lr, clipnorm=clipnorm) elif name == 'adagrad': lr = lr or 0.01 return optimizers.Adagrad(lr=lr, clipnorm=clipnorm) elif name == 'adam': lr = lr or 0.001 return optimizers.Adam(lr=lr, clipnorm=clipnorm) elif name == 'adamax': lr = lr or 0.001 return optimizers.Adamax(lr=lr, clipnorm=clipnorm) elif name == 'nadam': lr = lr or 0.001 return optimizers.Nadam(lr=lr, clipnorm=clipnorm) else: raise NotImplementedError
def getOptimizer(self, optimizer, options): if (optimizer == 'sgd'): return optimizers.SGD(lr=options[0], momentum=options[1], nesterov=options[2]) if (optimizer == 'adam'): return optimizers.Adam(lr=options[0], beta_1=options[1], beta_2=options[2], amsgrad=options[3]) if (optimizer == 'nadam'): return optimizers.Nadam(lr=options[0], beta_1=options[1], beta_2=options[2]) if (optimizer == 'rmsprop'): return optimizers.RMSprop(lr=options[0], rho=options[1])
def setOptimizer(self, config): configOptimizer = config["model"]["optimizer"].lower() if configOptimizer == "Adadelta".lower(): self.optimizer = optimizers.Adadelta() elif configOptimizer == "Adagrad".lower(): self.optimizer = optimizers.Adagrad() elif configOptimizer == "Adamax".lower(): self.optimizer = optimizers.Adamax() elif configOptimizer == "Ftrl".lower(): self.optimizer = optimizers.Ftrl() elif configOptimizer == "SGD".lower(): self.optimizer = optimizers.SGD() elif configOptimizer == "Nadam".lower(): self.optimizer = optimizers.Nadam() elif configOptimizer == "Optimizer".lower(): self.optimizer = optimizers.Optimizer() elif configOptimizer == "RMSprop".lower(): self.optimizer = optimizers.RMSprop()
def get_optimizer(): optimizer_name = optimizer_names[random.randint(0, len(optimizer_names) - 1)] model_attributes.optimizer_name = optimizer_name if optimizer_name == 'SGD': return optimizers.SGD(lr=get_learning_rate()) elif optimizer_name == 'RMSprop': return optimizers.RMSprop(lr=get_learning_rate()) elif optimizer_name == 'Adagrad': return optimizers.Adagrad(lr=get_learning_rate()) elif optimizer_name == 'Adadelta': return optimizers.Adadelta(lr=get_learning_rate()) elif optimizer_name == 'Adam': return optimizers.Adam(lr=get_learning_rate()) elif optimizer_name == 'Adamax': return optimizers.Adamax(lr=get_learning_rate()) elif optimizer_name == 'Nadam': return optimizers.Nadam(lr=get_learning_rate()) return None