def __init__(self, args): self.args = args self.model = FaceNet(args).model self.train_datasets, self.nrof_train = create_datasets_from_tfrecord( tfrcd_dir=args.datasets, batch_size=args.batch_size, phase='train') self.val_datasets, self.nrof_val = create_datasets_from_tfrecord( tfrcd_dir=args.datasets, batch_size=args.batch_size, phase='val') self.lr_schedule = schedules.ExponentialDecay(args.learning_rate, decay_steps=10000, decay_rate=0.96, staircase=True) self.optimizer = Adam(learning_rate=self.lr_schedule, beta_1=0.9, beta_2=0.999, epsilon=0.1) self.checkpoint = tf.train.Checkpoint( epoch=tf.Variable(0, dtype=tf.int64), n_iter=tf.Variable(0, dtype=tf.int64), best_pred=tf.Variable(0.0, dtype=tf.float32), optimizer=self.optimizer, model=self.model) self.manager = tf.train.CheckpointManager(self.checkpoint, args.checkpoint_dir, max_to_keep=3) check_folder(args.log_dir) self.train_summary_writer = tf.summary.create_file_writer(args.log_dir)
def get_learning_rate(self): """ Returns keras schedule for learning rate based on lr_schedule specified in the config :return: keras.optimizers.schedules """ if self.config.lr_schedule == 'ExponentialDecay': lr = schedules.ExponentialDecay( self.config.learning_rate, decay_steps=self.config.decay_steps, decay_rate=self.lr_decay, staircase=True) elif self.config.lr_schedule == 'PolynomialDecay': lr = schedules.PolynomialDecay(self.config.learning_rate, self.config.decay_steps, self.config.end_learning_rate, power=self.config.decay_power) elif self.config.lr_schedule == 'PiecewiseConstantDecay': lr = schedules.PiecewiseConstantDecay( self.config.piecewise_lr_boundaries, self.config.piecewise_lr_values) else: print("Invalid learning rate scheduler specified") raise ValueError return lr
def TrainKerasModel(path,model,opt,x,y,w,custom_callbacks=None): from tensorflow.keras.optimizers import SGD,Adam,RMSprop,schedules from tensorflow.keras.models import Sequential getopt = opt.split(',') optlist={} for o in getopt: osplit=o.split(';') if len(osplit) == 2: optlist[osplit[0].upper()]=osplit[1] else: print ('Your options has to be like following,') print ('opt1;val1,...,optn;valn') exit() lrate=float(optlist['LRATE']) momentum=float(optlist['MOMENTUM']) nepoch=int(optlist['EPOCH']) batch=int(optlist['BATCH']) optimizer=optlist['OPTIMIZER'] loss=optlist['LOSS'] lrate_dec=0 lrate_step=0 if 'LRATE_DEC' in optlist: lrate_dec=float(optlist['LRATE_DEC']) if 'LRATE_STEP' in optlist: lrate_step=int(optlist['LRATE_STEP']) #Metrics mets=[] #if loss == 'Exp': mets.append(ExponentialLoss_Metric) #Callbacks cbacks=[] if custom_callbacks != None: cbacks.append(custom_callbacks) if loss == 'Exp': loss = ExponentialLoss() if lrate_dec>0 and lrate_step>0: lrate = schedules.ExponentialDecay(initial_learning_rate=float(optlist['LRATE']), decay_steps=lrate_step, decay_rate=lrate_dec) print("Learning rate sceduled with initial rate "+str(float(optlist['LRATE']))+" by "+str(lrate_dec)+" decay and for each "+str(lrate_step)+" step.") optim=None if optimizer.upper() == 'SGD': optim=SGD(learning_rate=lrate, momentum=momentum) elif optimizer.upper() == 'ADAM': optim=Adam(learning_rate=lrate) elif optimizer.upper() == 'RMSPROP': optim=RMSprop(learning_rate=lrate, momentum=momentum) else: print ('Available optimizers: SGD,ADAM,RMSPROP') print ('You choose none, SGD will be run..') optim=SGD(learning_rate=lrate, momentum=momentum) model.compile(loss=loss, optimizer=optim, metrics=mets) print (model.summary()) model.fit(np.array(x).astype(np.float32), np.array(y).astype(np.float32), sample_weight = np.array(w).astype(np.float32), epochs=nepoch, batch_size=batch, callbacks=cbacks)
def __init__(self, initial_value: float, decay_steps: int, decay_rate: float, min_value=1e-4): super().__init__(schedule=schedules.ExponentialDecay(initial_value, decay_steps, decay_rate, staircase=True), min_value=min_value)
def __init__(self, initial_value: float, decay_steps: int, decay_rate: float, staircase=False, min_value=0.0): super().__init__(schedule=schedules.ExponentialDecay( initial_learning_rate=initial_value, decay_steps=decay_steps, decay_rate=decay_rate, staircase=staircase), min_value=min_value)
def setLRScheduler(self): """ Set Learning rate scheduler for training process. Returns ------- learning rate. """ lr_schedule = schedules.ExponentialDecay(self.lr_init_rate, decay_steps=100000, decay_rate=0.96, staircase=True) return lr_schedule
def styleTransfer(sourcepath, stylepath): path = 'static/uploads' base_image_path = os.path.join(path, "source.png") style_reference_image_path = os.path.join(path, "style.png") result_prefix = "static/results/result" global content_weight global total_variation_weight global style_weight total_variation_weight = 1e-6 style_weight = 3e-6 content_weight = 5e-7 width, height = image.load_img(base_image_path).size global img_nrows global img_ncols img_nrows = 400 img_ncols = int(width * img_nrows / height) content_img = preprocess_image(base_image_path) shape = content_img.shape[1:] model = VGG19(weights="imagenet", include_top=False, input_shape=shape) outputs_dict = dict([(layer.name, layer.output) for layer in model.layers]) global feature_extractor feature_extractor = Model(inputs=model.inputs, outputs=outputs_dict) global style_layer_names style_layer_names = [ "block1_conv1", "block2_conv1", "block3_conv1", "block4_conv1", "block5_conv1", ] global content_layer_name content_layer_name = "block5_conv2" optimizer = SGD( schedules.ExponentialDecay(initial_learning_rate=100.0, decay_steps=100, decay_rate=0.96)) base_image = preprocess_image(base_image_path) style_reference_image = preprocess_image(style_reference_image_path) combination_image = tf.Variable(preprocess_image(base_image_path)) iterations = 100 for i in range(1, iterations + 1): loss, grads = compute_loss_and_grads(combination_image, base_image, style_reference_image) optimizer.apply_gradients([(grads, combination_image)]) print("Iteration %d: loss=%.2f" % (i, (loss))) print(combination_image) img = deprocess_image(combination_image.numpy()) fname = result_prefix + ".png" image.save_img(fname, img)
def __init__(self, action_count, state_count, alpha=0.1, lr_decay_rate=0.9, steps=1000): self._state_count = state_count self._action_count = action_count self._model = self._build_model() # self._model.build() self._lr = schedules.ExponentialDecay(initial_learning_rate=alpha, decay_rate=lr_decay_rate, decay_steps=steps, staircase=True) self._optimizer = Adam(learning_rate=self._lr) self._model.compile(optimizer=self._optimizer, loss='mse')
def FCC_model(): inputs = Input(shape=(16, )) t = Dense(256)(inputs) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(256)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(256)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(128)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(128)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(128)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(128)(t) t = ReLU()(t) t = BatchNormalization()(t) t = Dense(64)(t) t = ReLU()(t) t = BatchNormalization()(t) outputs = Dense(11)(t) lr_schedule = schedules.ExponentialDecay(initial_learning_rate=5e-3, decay_steps=20000, decay_rate=0.9) model = Model(inputs, outputs) model.compile(optimizer=Adam(learning_rate=lr_schedule), loss=custom_loss) return model
def fit(self, model, metric, epochs, batch_size=256, learning_rate=0.1, callbacks=[], save_best=False): super().prepare_data(batch_size) self.epochs = epochs # run native fit lr_schedule = schedules.ExponentialDecay( initial_learning_rate=learning_rate, decay_steps=10000, decay_rate=0.95) if not callbacks: es_patience = 7 if self.searching_mode else 15 red_patience = 3 if self.searching_mode else 6 callbacks = [ tf.keras.callbacks.EarlyStopping(patience=es_patience, min_delta=1e-2), tf.keras.callbacks.ReduceLROnPlateau(patience=red_patience), tf.keras.callbacks.LearningRateScheduler(schedule=lr_schedule), tf.keras.callbacks.TerminateOnNaN(), ] if save_best: callbacks.append( tf.keras.callbacks.ModelCheckpoint(filepath='saved_models/s', save_weights_only=True, monitor='val_loss', mode='auto', verbose=0, save_best_only=True)) if self.save_tf_logs: log_dir = "logs/fit/" tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=log_dir) callbacks += [tensorboard_callback] # we will stop after a 1/3rd of total epochs to determine whether this model is performing so poorly # as to justify an early abort history = None for epochs in [round(epochs * .33), round(epochs * .66)]: history = model.fit(self.train_ds, validation_data=self.val_ds, epochs=epochs, batch_size=batch_size, verbose=1, callbacks=[callbacks]) score = history.history['val_mean_absolute_percentage_error'][-1] self.early_performance += [score] if len(self.early_performance) > 10: benchmark = median(self.early_performance ) # median ignore crazy excessive outliers if score > benchmark: add_to_log("Performing poorly, discontinuing") break # return metrics self.model = model self.history = history return self.history.history[metric][-1]
#model.add(Dense(2*numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dense(2*numfeatures, kernel_initializer=initializer1, activation='relu')) #model.add(Dense(4*numfeatures, kernel_initializer=initializer1, activation='relu')) #model.add(Dense(4*numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dense(4*numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dense(4*numfeatures, kernel_initializer=initializer1, activation='relu')) #model.add(Dense(2*numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dense(2*numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dense(numfeatures, kernel_initializer=initializer1, activation='relu')) model.add(Dropout(0.1)) model.add(Dense(10, kernel_initializer=initializer1, activation='relu')) model.add(Dense(len(df.EleType.unique()), kernel_initializer='glorot_uniform', activation='softmax')) es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10) lr_schedule = schedules.ExponentialDecay(initial_learning_rate=1e-3,decay_steps=10000,decay_rate=0.9) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=lr_schedule), metrics=['accuracy',]) # In[14]: history = model.fit(X_train,y_train,validation_data=(X_test,y_test,Wt_test),epochs=epochs,batch_size=batch_size,shuffle=True,sample_weight=Wt_train,callbacks=[es]) # In[15]: model.save(plot_dir+'my_model.h5') fig, ax = plt.subplots(1, 1, figsize=(5, 5))
def __init__( self, environment_spec: specs.EnvironmentSpec, network: snt.RNNCore, counter: counting.Counter = None, logger: loggers.Logger = None, snapshot_dir: Optional[str] = None, n_step_horizon: int = 16, minibatch_size: int = 80, learning_rate: float = 2e-3, discount: float = 0.99, gae_lambda: float = 0.99, decay: float = 0.99, epsilon: float = 1e-5, entropy_cost: float = 0., baseline_cost: float = 1., max_abs_reward: Optional[float] = None, max_gradient_norm: Optional[float] = None, max_queue_size: int = 100000, verbose_level: Optional[int] = 0, ): # Internalize spec and replay buffer. self._environment_spec = environment_spec self._verbose_level = verbose_level self._minibatch_size = minibatch_size self._queue = queue.ReplayBuffer(max_queue_size=max_queue_size, batch_size=n_step_horizon) # Internalize network. self._network = network # Setup optimizer and learning rate scheduler. self._learning_rate = tf.Variable(learning_rate, trainable=False) self._lr_scheduler = schedules.ExponentialDecay( initial_learning_rate=learning_rate, decay_steps=8000, # TODO make a flag decay_rate=0.96 # TODO make a flag ) self._optimizer = snt.optimizers.RMSProp( learning_rate=self._learning_rate, decay=decay, epsilon=epsilon) #self._optimizer = snt.optimizers.Adam( # learning_rate=self._learning_rate, #) # Hyperparameters. self._discount = discount self._gae_lambda = gae_lambda self._entropy_cost = entropy_cost self._baseline_cost = baseline_cost # Set up reward/gradient clipping. if max_abs_reward is None: max_abs_reward = np.inf if max_gradient_norm is None: max_gradient_norm = 1e10 # A very large number. Infinity results in NaNs. self._max_abs_reward = tf.convert_to_tensor(max_abs_reward) self._max_gradient_norm = tf.convert_to_tensor(max_gradient_norm) if snapshot_dir is not None: self._snapshotter = tf2_savers.Snapshotter( objects_to_save={'network': self._network}, directory=snapshot_dir, time_delta_minutes=60.) # Logger. self._counter = counter or counting.Counter() self._logger = logger # Do not record timestamps until after the first learning step is done. # This is to avoid including the time it takes for actors to come online and # fill the replay buffer. self._timestamp = None self._pi_old = None
checkpoint_path = "Regression_Model/" + trainid + "-{epoch:04d}.ckpt" # Create a callback that saves the model's weights cp_callback = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1, period=10) model = Regression_CNN(length) log_dir = "TensorBoard/Regression/" + trainid + "_" + datetime.now( ).strftime("%Y%m%d-%H%M%S") tensorboard_cbk = TensorBoard(log_dir=log_dir) lr_schedule = schedules.ExponentialDecay(learning_rate, decay_steps=5000, decay_rate=0.96) model.compile(loss='mean_squared_error', optimizer=SGD(momentum=0.98, learning_rate=lr_schedule), metrics=['mse']) #model.compile(loss='mean_absolute_error', optimizer= SGD(momentum = 0.98, learning_rate = lr_schedule), metrics=['mse']) #plot_model(model, to_file=combination+'model_plot.png', show_shapes=True, show_layer_names=True) #print(model.summary()) model.fit(training_generator, epochs=epochs, validation_data=validation_generator, callbacks=[tensorboard_cbk, cp_callback]) evaluate(train_x, train_y, train_pasid, trainid, 'train', label_mean,
def train_top(model_name, load_weights_path=None): backbone, model = model_inceptionv3() # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional InceptionV3 layers for layer in backbone.layers: layer.trainable = True # compile the model (should be done *after* setting layers to non-trainable) lr_schedule = schedules.ExponentialDecay( initial_learning_rate=Config.LEARNING_RATE, decay_steps=Config.DECAY_STEPS, decay_rate=Config.DECAY_RATE) optimizer = Adam(learning_rate=lr_schedule) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) plot_model(model, show_shapes=True, to_file='inceptionv3_architecture.png') exit() if load_weights_path is not None: print(f"Loaded weights {load_weights_path}") model.load_weights(load_weights_path) model_path = os.path.join("models", model_name + "-{epoch:02d}-{val_loss:.3f}.hdf5") df = pd.read_csv(Config.TRAINING_CSV_PATH) df["label"] = df["label"].apply( lambda x: str(np.argmax(np.array(x[1:-1].split(","), dtype="int")))) num_samples = np.bincount(df["label"]) class_weight = {} for i, num in enumerate(num_samples): class_weight[i] = 1. / num datagen = ImageDataGenerator( preprocessing_function=inception_v3.preprocess_input, validation_split=Config.VALIDATION_RATIO) train_generator = datagen.flow_from_dataframe( subset="training", dataframe=df, directory=Config.TRAINING_IMAGES_PATH, x_col="image", y_col="label", class_mode="categorical", batch_size=Config.BATCH_SIZE, shuffle=True) validation_generator = datagen.flow_from_dataframe( subset="validation", dataframe=df, directory=Config.TRAINING_IMAGES_PATH, x_col="image", y_col="label", class_mode="categorical", batch_size=Config.BATCH_SIZE, shuffle=False) checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=1, save_best_only=False) tensorboard = TensorBoard( log_dir=f"./logs/{model_name}-{int(time.time())}", update_freq=1) earlystopping = EarlyStopping(monitor='val_loss', patience=20, min_delta=0.01) model.fit(train_generator, initial_epoch=Config.INITIAL_EPOCH, steps_per_epoch=train_generator.samples // Config.BATCH_SIZE, validation_data=validation_generator, validation_steps=validation_generator.samples // Config.BATCH_SIZE, epochs=Config.TRAIN_EPOCH, callbacks=[checkpoint, tensorboard, earlystopping])