def build_model(hp): params = hp.copy() params['e_dim'] = params['dim'] params['r_dim'] = params['dim'] params['name'] = 'embedding_model' embedding_model = models[params['embedding_model']] embedding_model = embedding_model(**params) triple = Input((3, )) ftriple = Input((3, )) inputs = [triple, ftriple] score = embedding_model(triple) fscore = embedding_model(ftriple) loss_function = loss_function_lookup(params['loss_function']) loss = loss_function(score, fscore, params['margin'] or 1, 1) model = Model(inputs=inputs, outputs=loss) model.add_loss(loss) model.compile(optimizer=Adam(learning_rate=ExponentialDecay( params['learning_rate'], decay_steps=100000, decay_rate=0.96)), loss=None) return model
def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float, learning_rate_decay_steps: int, ) -> tf_optimizers.Optimizer: # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD(learning_rate=learning_rate_schedule) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop(learning_rate=learning_rate_schedule) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def siamese_model_ResNet(model_type): if model_type == 'ResNet_50': LR = 0.001 base_model = ResNet_50() if model_type == 'ResNet_18': LR = 0.0001 base_model = ResNet_18() imgA = Input(shape=(64, 64, 3)) imgB = Input(shape=(64, 64, 3)) featsA = base_model(imgA) featsB = base_model(imgB) distance = Lambda(euclidean_distance, name='compute_ED')([featsA, featsB]) model = Model(inputs=[imgA, imgB], outputs=distance, name='Contrastive') # model.load_weights('saved_models/checkpoints/' + model_name) initial_learning_rate = LR lr_schedule = ExponentialDecay(initial_learning_rate, decay_steps=1562, decay_rate=0.90, staircase=True) model.compile(loss=contrastive_loss, optimizer=keras.optimizers.Adam(lr_schedule), metrics=[accuracy1]) #, recall_m, precision_m]) model.summary() return model
def __init__(self, num_scales, num_iters, max_size, min_size, scale_factor, learning_rate, checkpoint_dir, debug): self.num_scales = num_scales self.num_iters = num_iters self.num_filters = [ 32 * pow(2, (scale // 4)) for scale in range(self.num_scales) ] # num_filters double for every 4 scales self.max_size = max_size self.min_size = min_size self.scale_factor = scale_factor self.noise_amp_init = 0.1 self.checkpoint_dir = checkpoint_dir self.G_dir = self.checkpoint_dir + '/G' self.D_dir = self.checkpoint_dir + '/D' self.learning_schedule = ExponentialDecay( learning_rate, decay_steps=4800, decay_rate=0.1, staircase=True) # 1600 * 3 steps self.build_model() self.debug = debug if self.debug: self.create_summary_writer() self.create_metrics()
def _compile_model( self, loss_function: Callable, ground_distance_path: Path, **loss_function_kwargs ): if loss_function == self_guided_earth_mover_distance: self.emd_weight_head_start = EmdWeightHeadStart() self.ground_distance_manager = GroundDistanceManager(ground_distance_path) lr_schedule = ExponentialDecay( self.learning_rate, decay_steps=429, decay_rate=0.995 ) self.compile( loss=loss_function( model=self, **loss_function_kwargs ), optimizer=self._OPTIMIZER( learning_rate=lr_schedule, nesterov=True, momentum=self._OPTIMIZER_MOMENTUM ), metrics=self._METRICS, run_eagerly=True )
def get_lr_scheduler(learning_rate, decay_type, decay_steps): if decay_type: decay_type = decay_type.lower() if decay_type == None: lr_scheduler = learning_rate elif decay_type == 'cosine': lr_scheduler = CosineDecay( initial_learning_rate=learning_rate, decay_steps=decay_steps, alpha=0.2) # use 0.2*learning_rate as final minimum learning rate elif decay_type == 'exponential': lr_scheduler = ExponentialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, decay_rate=0.9) elif decay_type == 'polynomial': lr_scheduler = PolynomialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, end_learning_rate=learning_rate / 100) elif decay_type == 'piecewise_constant': #apply a piecewise constant lr scheduler, including warmup stage boundaries = [500, int(decay_steps * 0.9), decay_steps] values = [ 0.001, learning_rate, learning_rate / 10., learning_rate / 100. ] lr_scheduler = PiecewiseConstantDecay(boundaries=boundaries, values=values) else: raise ValueError('Unsupported lr decay type') return lr_scheduler
def define_model(): model = Sequential() model.add( Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3))) model.add(BatchNormalization()) model.add( Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same')) model.add(BatchNormalization()) model.add(MaxPooling2D((2, 2))) model.add(Dropout(0.2)) model.add( Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same')) model.add(BatchNormalization()) model.add( Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same')) model.add(BatchNormalization()) model.add(MaxPooling2D((2, 2))) model.add(Dropout(0.3)) model.add( Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same')) model.add(BatchNormalization()) model.add( Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same')) model.add(BatchNormalization()) model.add(MaxPooling2D((2, 2))) model.add(Dropout(0.4)) model.add(Flatten()) model.add(Dense(128, activation='relu', kernel_initializer='he_uniform')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) # compile model lr_schedule = ExponentialDecay(initial_learning_rate=0.01, decay_steps=10000, decay_rate=0.9) opt = SGD(learning_rate=lr_schedule) model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) return model
def getScheduler(sched, lr=1e-3): if sched == 'poly': return PolynomialDecay(lr, 3000, 0.99) elif sched == 'exp': return ExponentialDecay(lr, 1000) else: return lr
def train_from_cache( *, architecture: str = "BiT", size: int = 256, augmentation: int = 1, epochs: int = 20, batch_size: int = 32, learning_rate: float = 0.001, lr_decay_rate: float = 0.99, lr_decay_steps: int = 5e2, **model_params, ) -> float: """ Trains the model using embeddings that were previously cached. """ train_data, dev_data = load_embeddings(architecture, size, augmentation) # Make and init the wandb run. wandb.init(project="Plant Pathology", reinit=True) wandb.config.update( { "architecture": architecture, "epochs": epochs, "batch_size": batch_size, "learning_rate": learning_rate, "lr_decay_rate": lr_decay_rate, "lr_decay_steps": lr_decay_steps, "augmentation": augmentation, "size": size, } ) model = PathologistModel(nclasses=constants.NCLASSES, **model_params) model.compile( optimizer=Adam( learning_rate=ExponentialDecay(learning_rate, lr_decay_steps, lr_decay_rate) ), loss="categorical_crossentropy", metrics=["categorical_accuracy"], ) model.fit( train_data["X"], train_data["y"], batch_size=batch_size, epochs=epochs, validation_data=(dev_data["X"], dev_data["y"]), callbacks=[WandbCallback(save_model=False)], ) # Log the scores train_loss, train_acc = model.evaluate(train_data["X"], train_data["y"]) _, dev_acc = model.evaluate(dev_data["X"], dev_data["y"]) wandb.run.summary.update( {"final_train_loss": train_loss, "final_train_acc": train_acc} ) wandb.join() return dev_acc
def test_constant_lr(self): """Test a constant learning rate""" lrs = ExponentialDecay( initial_learning_rate=0.1, decay_steps=10000000, decay_rate=1.0, ) for i in range(25): assert round(float(lrs(i).numpy()), 5) == 0.1
def __init__(self, model, ckpt_path, args): super().__init__(model=model, loss=MeanAbsoluteError(), learning_rate=ExponentialDecay(args.lr_init, args.lr_decay_step, args.lr_decay_ratio, staircase=True), checkpoint_path=ckpt_path, args=args)
def get_optimizer( optimizer_key: str, learning_rate: float, learning_rate_decay: float = 1.0, learning_rate_decay_steps: int = 1000000, gradient_clip_value: float = 1000000, ) -> tf_optimizers.Optimizer: """ This function defines the optimizer used by ml4ir. Users have the option to define an ExponentialDecay learning rate schedule Arguments: optimizer_key: string optimizer name to be used as defined under ml4ir.base.config.keys.OptimizerKey learning_rate: floating point learning rate for the optimizer learning_rate_decay: floating point rate at which the learning rate will be decayed every learning_rate_decay_steps learning_rate_decay_steps: int representing number of iterations after which learning rate will be decreased exponentially gradient_clip_value: float value representing the clipvalue for gradient updates. Not setting this to a reasonable value based on the model will lead to gradient explosion and NaN losses. References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay FIXME: Define all arguments overriding tensorflow defaults in a separate file for visibility with ml4ir users """ # Define an exponential learning rate decay schedule learning_rate_schedule = ExponentialDecay( learning_rate, decay_steps=learning_rate_decay_steps, decay_rate=learning_rate_decay, staircase=True, ) if optimizer_key == OptimizerKey.ADAM: return tf_optimizers.Adam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.NADAM: return tf_optimizers.Nadam( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.ADAGRAD: return tf_optimizers.Adagrad( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.SGD: return tf_optimizers.SGD( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) elif optimizer_key == OptimizerKey.RMS_PROP: return tf_optimizers.RMSprop( learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value ) else: raise ValueError("illegal Optimizer key: " + optimizer_key)
def __init__(self, env: '', episodes=1000, alpha=0.01, gamma=0.9, alpha_decay_rate=0.9): self.env = Environment(env=env) self.episodes = episodes self.lr = ExponentialDecay(alpha, episodes, alpha_decay_rate) self.optimizer = Adam(self.lr) self.action_count, self.states_count = self.env.spaces_count() self.gamma = gamma self._net = ReinforcePolicyNet(action_count=self.action_count, states_count=self.states_count) self._model = ReinforcePolicyModel(self._net) self._agent = ReinforcePolicyAgent(env=self.env, model=self._model, gamma=gamma) self.huber_loss = Huber(reduction=tf.keras.losses.Reduction.SUM)
def nvidia(): """ Implementation of Nvidia's End-to-End Learning model for Self-driving cars """ global X_train, y_train # Model Design inputs = Input(shape=(160,320,3)) cropped = Cropping2D(cropping=((64, 0), (0, 0)))(inputs) resized_input = Lambda(lambda image: tf.image.resize(image, (66,200)))(cropped) normalize_layer = LayerNormalization(axis=1)(resized_input) conv1 = Conv2D(filters=24, kernel_size=5, strides=(2,2), activation='relu')(normalize_layer) conv2 = Conv2D(filters=36, kernel_size=5, strides=(2,2), activation='relu')(conv1) conv3 = Conv2D(filters=48, kernel_size=5, strides=(2,2), activation='relu')(conv2) conv4 = Conv2D(filters=64, kernel_size=3, activation='relu')(conv3) conv5 = Conv2D(filters=64, kernel_size=3, activation='relu')(conv4) flatten = Flatten()(conv5) dense1 = Dense(100,activation='relu')(flatten) dense2 = Dense(50,activation='relu')(dense1) dense3 = Dense(10,activation='relu')(dense2) out = Dense(1, activation='linear')(dense3) # Specifications and training checkpoint = ModelCheckpoint(filepath="./ckpts/model_nvidia.h5", monitor='val_loss', save_best_only=True) stopper = EarlyStopping(monitor='val_loss', min_delta=0.0003, patience = 10) lr_schedule = ExponentialDecay(initial_learning_rate=0.0001, decay_steps=100000, decay_rate=0.95) optimizer = Adam(learning_rate=lr_schedule) loss = Huber(delta=0.5, reduction="auto", name="huber_loss") t2 = time() model = Model(inputs=inputs, outputs=out) model.compile(loss = loss, optimizer = optimizer) result = model.fit(X_train, y_train, validation_split = 0.2, shuffle = True, epochs = 100, callbacks=[checkpoint, stopper]) # Visualization of loss variations across epochs plt.plot(result.history['loss']) plt.plot(result.history['val_loss']) plt.title('Huber Loss') plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Training set', 'Validation set'], loc = 'upper right') plt.savefig('loss.png') plt.show() print("Time taken to train: {:.2f}s".format(time()-t2)) model.load_weights('./ckpts/model_nvidia.h5') model.save('model.h5')
def __init__(self, config: MuZeroConfig, storage: SharedStorage, replay_buffer: ReplayBuffer): self.config = config self.storage = storage self.replay_buffer = replay_buffer self.summary = create_summary(name="leaner") self.metrics_loss = Mean(f'leaner-loss', dtype=tf.float32) self.network = Network(self.config) self.lr_schedule = ExponentialDecay( initial_learning_rate=self.config.lr_init, decay_steps=self.config.lr_decay_steps, decay_rate=self.config.lr_decay_rate) self.optimizer = Adam(learning_rate=self.lr_schedule)
def exponential(learn_rate): if not isinstance(learn_rate, (list, tuple)): raise ValueError( "Exponential decay requres three values: starting learning rate, steps, and decay factor" ) start_lr = learn_rate[0] steps = learn_rate[1] decay = learn_rate[2] learning_rate_sced = ExponentialDecay(start_lr, steps, decay, staircase=True) return learning_rate_sced
def transfer_train( *, train_set: str, size: int = 256, epochs: int = 20, batch_size: int = 32, learning_rate: float = 0.001, lr_decay_rate: float = 0.99, lr_decay_steps: int = 5e2, make_submission: bool = False, **model_params, ): """ Trains a new head on top of a transfer model. No fine tuning of the transfer model is conducted. Transfer model embeddings are computed once at the beginning of the training run. If `make_submission==True`, returns this model's scores on the test set, along with the corresponding image ids. """ # Make and init the wandb run. wandb.init(project="Plant Pathology", reinit=True) wandb.config.update({ "epochs": epochs, "batch_size": batch_size, "learning_rate": learning_rate, "lr_decay_rate": lr_decay_rate, "lr_decay_steps": lr_decay_steps, "size": size, }) model = TransferModel(constants.NCLASSES, size, batch_size, **model_params) model.compile( optimizer=Adam(learning_rate=ExponentialDecay( learning_rate, lr_decay_steps, lr_decay_rate)), loss="categorical_crossentropy", metrics=["categorical_accuracy"], ) # Train the model (just the new layers on top of the transfer model) model.fit_head( train_set, "dev", epochs=epochs, callbacks=[WandbCallback(save_model=False)], ) # Log the scores wandb.join() if make_submission: return model.predict_on_test()
def get_network(learning_rate=0.001, momentum=0.5, decay_rate=0.98, use_sgd=True, kernel_regularizer_conv=0.01, kernel_regularizer_dense=0.0001, **kwargs): convolutional_network = _get_convolutional_network( kernel_regularizer_conv, kernel_regularizer_dense) input_image_1 = Input(input_shape) input_image_2 = Input(input_shape) output_image_1 = convolutional_network(input_image_1) output_image_2 = convolutional_network(input_image_2) l1_distance_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1])) l1_distance = l1_distance_layer([output_image_1, output_image_2]) prediction = Dense(units=1, activation='sigmoid', kernel_initializer=RandomNormal(mean=0, stddev=0.01), bias_initializer=RandomNormal(mean=0.5, stddev=0.01))(l1_distance) model = Model(inputs=[input_image_1, input_image_2], outputs=prediction) if use_sgd: optimizer = SGD(learning_rate=ExponentialDecay(learning_rate, 100000, decay_rate), momentum=momentum) else: optimizer = Adam( learning_rate=ExponentialDecay(learning_rate, 100000, decay_rate)) model.compile(loss='binary_crossentropy', metrics=['binary_accuracy'], optimizer=optimizer) return model
def train(args): awv = audio_array( args.content) #get waveform array from folder containing wav files aspec = tospec(awv) #get spectrogram array adata = splitcut(aspec) #split spectrogams to fixed length bwv = audio_array(args.style) bspec = tospec(bwv) bdata = splitcut(bspec) dsa = tf.data.Dataset.from_tensor_slices(adata).repeat(50).map( lambda x: proc(x, hp), num_parallel_calls=tf.data.experimental.AUTOTUNE).shuffle(10000).batch( hp.bs, drop_remainder=True) dsb = tf.data.Dataset.from_tensor_slices(bdata).repeat(50).map( lambda x: proc(x, hp), num_parallel_calls=tf.data.experimental.AUTOTUNE).shuffle(10000).batch( hp.bs, drop_remainder=True) dataset = tf.data.Dataset.zip((dsa, dsb)) model = Adverserial() gen_opt = Adam( ExponentialDecay(hp.lr, decay_steps=hp.epochs * 2, decay_rate=0.1), hp.opt_alpha) disc_opt = Adam( ExponentialDecay(hp.lr, decay_steps=hp.epochs * 2, decay_rate=0.1), hp.opt_alpha) model.compile(gen_opt, disc_opt, adv_loss) # Log metrics with wandb callbacks = [ TrainCallback(model, aspec), EarlyStopping(monitor="loss_g", patience=50) ] model.fit(dataset, epochs=hp.epochs, callbacks=callbacks)
def main(args): timestamp = int(time.time()) logdir = os.path.join(args.volumedir, datetime.datetime.today().strftime('%Y%m%d'), args.logdir) if not os.path.isdir(logdir): os.makedirs(logdir) snapdir = os.path.join(args.volumedir, datetime.datetime.today().strftime('%Y%m%d'), "progess_snaps") if not os.path.isdir(snapdir): os.makedirs(snapdir) hdlr = logging.FileHandler( os.path.join(logdir, "training_output_%d.log" % timestamp)) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.INFO) checkpointdir = os.path.join(args.volumedir, datetime.datetime.today().strftime('%Y%m%d'), args.checkpointdir) if not os.path.isdir(checkpointdir): os.makedirs(checkpointdir) # Dynamically load modelBuilder class moduleName, klassName = args.modelbuilder.split(".") mod = __import__('models.%s' % moduleName, fromlist=[klassName]) klass = getattr(mod, klassName) modelBuilder = klass(args) optimizer_map = {"adam": Adam, "rmsprop": RMSprop, "sgd": SGD} optimizer = optimizer_map[ args.optimizer] if args.optimizer in optimizer_map.keys() else RMSprop lr_decay = ExponentialDecay(initial_learning_rate=args.learningrate, decay_rate=args.decayrate, decay_steps=args.decaysteps) opt = optimizer(learning_rate=lr_decay, clipvalue=3) model = modelBuilder.get_model() C, S, G = modelBuilder.get_train_data() for i in range(1, args.numepochs + 1): loss, grads = modelBuilder.compute_loss_and_grads(model, C, S, G) opt.apply_gradients([(grads, G)]) if i % 100 == 0: logger.info("Iteration: %d, Loss: %.2f" % (i, loss)) modelBuilder.save_generated_img(snapdir, G.numpy(), i, timestamp)
def getModel(lag): model = Sequential([ Dense(16, activation='relu'), Dense(32, activation='relu'), Dense(16, activation='relu'), Dense(1, activation='linear') ]) model.build(input_shape=(None, lag)) model.compile(optimizer=Adam(ExponentialDecay(1e-3, 50, 0.9)), loss=BinaryCrossentropy(from_logits=True)) return model
def test_exp_lr(self): """Test a exponential learning rate""" gold = [0.10000000149011612,0.09486832469701767,0.08999999612569809,0.08538150042295456,0.08099999278783798, 0.07684334367513657,0.07289998978376389,0.06915900856256485,0.06560999155044556,0.062243103981018066, 0.05904899165034294,0.05601879581809044,0.05314409360289574,0.050416912883520126,0.04782968387007713, 0.045375220477581024,0.04304671287536621,0.04083769768476486,0.03874203935265541,0.036753926426172256, 0.03486783429980278,0.03307853266596794,0.03138105198740959,0.029770677909255028,0.028242945671081543] lrs = ExponentialDecay( initial_learning_rate=0.1, decay_steps=2, decay_rate=0.9, ) scheduler = [lrs(i) for i in range(25)] self.assertTrue(self.compare_lr_values(scheduler, gold))
def compile_model(input_timesteps: int = 1, output_timesteps: int = 1, categories: int = 3, loss_function: str = "categorical_crossentropy", initial_learning_rate: float = 0.001, decay_steps: int = 1000, decay_rate: float = 0.95, metrics: list = [], **kwargs): model = Sequential() model.add(Conv3D(filters=16, kernel_size=(3, 3, 3), padding="same")) model.add( Bidirectional( ConvLSTM2D(filters=16, kernel_size=(3, 3), return_sequences=True, padding="same", dropout=0.1))) model.add(BatchNormalization()) model.add( Bidirectional( ConvLSTM2D(filters=32, kernel_size=(3, 3), return_sequences=True, padding="same", dropout=0.5))) model.add(BatchNormalization()) model.add( Bidirectional( ConvLSTM2D(filters=32, kernel_size=(3, 3), return_sequences=True, padding="same", dropout=0.5))) model.add(BatchNormalization()) model.add( Conv3D(filters=categories, kernel_size=(input_timesteps - output_timesteps + 1, 1, 1), activation="softmax", padding="valid")) optimizer = Adam(learning_rate=ExponentialDecay( initial_learning_rate=initial_learning_rate, decay_steps=decay_steps, decay_rate=decay_rate)) model.compile(loss=loss_function, optimizer=optimizer, metrics=metrics) return model
def get_lr_scheduler(learning_rate, decay_type, decay_steps): if decay_type: decay_type = decay_type.lower() if decay_type == None: lr_scheduler = learning_rate elif decay_type == 'cosine': lr_scheduler = CosineDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps) elif decay_type == 'exponential': lr_scheduler = ExponentialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, decay_rate=0.9) elif decay_type == 'polynomial': lr_scheduler = PolynomialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, end_learning_rate=learning_rate/100) else: raise ValueError('Unsupported lr decay type') return lr_scheduler
def getExtremeModel(optimizerExtremeParam, lag): modelExtreme = Sequential([ Dense(16, activation='relu'), Dense(32, activation='relu'), Dense(64, activation='relu'), Dense(32, activation='relu'), Dense(16, activation='relu'), Dense(1, activation='sigmoid'), ]) modelExtreme.build(input_shape=(None, lag)) modelExtreme.compile(optimizer=Adam( ExponentialDecay(*optimizerExtremeParam)), loss=tf.losses.MeanSquaredError()) return modelExtreme
def getNormalModel(optimizerNormalParam, lag): modelNormal = Sequential([ Dense(16, activation='relu'), Dense(32, activation='relu'), Dense(64, activation='relu'), Dense(32, activation='relu'), Dense(16, activation='relu'), Dense(1, activation='linear'), ]) modelNormal.build(input_shape=(None, lag)) modelNormal.compile(optimizer=Adam( ExponentialDecay(*optimizerNormalParam)), loss=MeanSquaredError()) return modelNormal
def train_model(model, x, y, task): date = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") checkpoints_path = f'/opt/ml/model/checkpoints/{task}/{date}' checkpoints_file_format = 'checkpoints.{epoch:02d}-{val_loss:.2f}.hdf5' checkpoint_callback = ModelCheckpoint( posixpath.join(checkpoints_path, checkpoints_file_format)) L = len(x) split_ration = 0.8 train_indexes = random.sample(range(L), int(L * split_ration)) validation_indexes = list(set(range(L)) - set(train_indexes)) x_train = x[train_indexes] y_train = y[train_indexes] x_val = x[validation_indexes] y_val = y[validation_indexes] generator = ImageDataGenerator(rotation_range=30, zoom_range=0.20, fill_mode="nearest", shear_range=0.20, horizontal_flip=False, width_shift_range=0.2, height_shift_range=0.2) lr_schedule = ExponentialDecay(initial_learning_rate=0.01, decay_steps=100, decay_rate=0.9, staircase=False) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=lr_schedule), metrics=['accuracy']) model.fit(generator.flow(x_train, y_train, batch_size=32), steps_per_epoch=len(x_train) // 32, verbose=2, epochs=500, shuffle=True, validation_data=(x_val, y_val), callbacks=[ WandbCallback(), checkpoint_callback, LearningRateCallback(model) ]) model.save('/opt/ml/model')
def get_model(n_categories, net_spec, learning_rate, steps_per_epoch, lr_decay_period=5, staircase=True): """ Generates a sequential model with the provided parameters Args: n_categories: number of possible outcomes for the model to predict net_spec: a tuple with the numbers of nodes in each layer. For single layer must be like (N,) learning_rate: initial learning rate for the optimization process steps_per_epoch: total steps taken in a single epoch lr_decay_period: number of epochs before the learning rate becomes halved staircase: boolean. Whether to use a staircase pattern for the learning rate decay Returns: A corresponding keras.Sequetial model """ model = keras.Sequential([ layers.Dense(net_spec[0], input_dim=13, activation='relu', name='hidden_layer_0'), ] + [ layers.Dense(layer_size, activation='relu', name=f'hidden_layer_{i+1}') for i, layer_size in enumerate(net_spec[1:]) ] + [ layers.Dense(n_categories, activation='softmax', name='output_layer') ]) decay_steps = steps_per_epoch * lr_decay_period lr_schedule = ExponentialDecay(learning_rate, decay_steps=decay_steps, decay_rate=0.5, staircase=staircase) opt = keras.optimizers.Adam(learning_rate=lr_schedule) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['categorical_accuracy']) return model
def create_model(input_shape=10, output_shape=10, layers=[50], activation='relu', optimizer='adam', learning_rate=0.001, dropout=True, decay=0.9, decay_steps=10000, batch_norm=True): model = Sequential(name=str(time.time())) model.add(Input(input_shape)) for l in layers: model.add(Dense(l)) if batch_norm: model.add(BatchNormalization()) model.add(Activation(activation=activation)) if dropout: model.add(Dropout(0.5)) model.add(Dense(output_shape)) # Compile model lr_schedule = ExponentialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, decay_rate=decay) opt = optimizers.get({ 'class_name': optimizer, 'config': { 'learning_rate': lr_schedule } }) model.compile(loss='mse', optimizer=opt, metrics=[r2_metric]) return model
def training(self): def _loss(y_true, y_pred): print('y_pred.shape:', y_pred.shape) print('y_true.shape: ', y_true.shape) labels = tf.cast(tf.squeeze(y_true), tf.int64) return tf.reduce_mean( input_tensor=tf.nn.sparse_softmax_cross_entropy_with_logits( logits=y_pred, labels=labels)) print('Executing eagerly? ', tf.executing_eagerly()) self.build_graph() learning_rate = ExponentialDecay(initial_learning_rate=LEARNING_RATE, decay_steps=2000, decay_rate=0.97, staircase=True) optimizer = optimizers.Adam(learning_rate=learning_rate) #self.model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"]) self.model.compile(loss=_loss, optimizer=optimizer, metrics=["accuracy"]) self.model.run_eagerly = True self.model.summary() #keras.utils.plot_model(self.model, show_shapes=True) checkpoint_callback = callbacks.ModelCheckpoint(MODEL_NAME + ".h5", save_freq='epoch') time_callback = TimeHistory() run_logdir = self.get_run_logdir() tensorboard_callback = callbacks.TensorBoard(run_logdir) data = Data() train_dataset = data.prepare_data_for_training(Data.DATA_TRAINING, batch_size) history = self.model.fit(train_dataset, steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[time_callback]) #score = self.model.evaluate(X_test, y_test) self.save_model()