def rmse(y_true, y_pred): m_factor = rmse_factor m = RootMeanSquaredError() m.update_state(inverse_transform(y_true, mmn), inverse_transform(y_pred, mmn)) # return denormalize(m.result().numpy(), mmn) * m_factor return m.result().numpy() * m_factor
def build_model(self): input_layer = Input((self.ts_length, self.num_variables)) conv1 = Conv1D(filters=128, kernel_size=8, padding='same')(input_layer) conv1 = BatchNormalization()(conv1) conv1 = Activation(activation='relu')(conv1) conv2 = Conv1D(filters=256, kernel_size=5, padding='same')(conv1) conv2 = BatchNormalization()(conv2) conv2 = Activation('relu')(conv2) conv3 = Conv1D(128, kernel_size=3, padding='same')(conv2) conv3 = BatchNormalization()(conv3) conv3 = Activation('relu')(conv3) gap_layer = GlobalAveragePooling1D()(conv3) output_layer = Dense(self.num_variables, activation='linear')(gap_layer) model = Model(inputs=input_layer, outputs=output_layer) model.compile(loss=self.loss, optimizer=self.optimizer, metrics=[RootMeanSquaredError()]) return model
def build_cnn(n_hidden=2, n_neuron=50, learning_rate=1e-3, in_shape=200, drop=0.1): """ Objective: Create Convolutional Neural Network Architecture for regression How to get started: https://www.datatechnotes.com/2019/12/how-to-fit-regression-data-with-cnn.html How to tune: https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/ Valuable info: https://github.com/keras-team/keras/blob/8a8ef43ffcf8d95d2880da073bbcee73e51aad48/docs/templates/getting-started/sequential-model-guide.md :param input_shape: :return: """ model = keras.models.Sequential() # Experiment on both Conv1D and Conv2D is needed. # According to most tutorial, the input_shape depends on the shape of your data. In this case, the shape of our data # is (number_of_features, 1) since model.add(keras.layers.Dropout(drop, input_shape=(in_shape, 1))) # in_shape should be iterable (tuple) for layer in range(n_hidden): # create hidden layers model.add(keras.layers.Dense(n_neuron, activation="relu")) model.add(keras.layers.Dropout(drop)) # add dropout to model after the a dense layer model.add(Conv1D(32, 2, activation='relu', input_shape=(in_shape, 1))) # model.add(keras.layers.MaxPooling1D(pool_size=3)) model.add(Conv1D(64, 2, activation='relu', input_shape=(in_shape, 1))) model.add(Flatten(input_shape=(in_shape, 1))) model.add(Dense(64, activation="relu")) model.add(Dense(1)) optimizer = keras.optimizers.Adam(learning_rate=learning_rate) model.compile(loss="mse", optimizer=optimizer, metrics=[RootMeanSquaredError(name='rmse')]) return model
def build_nn(n_hidden=2, n_neuron=50, learning_rate=1e-3, in_shape=200, drop=0.1): """ Create neural network architecture and compile. Accepts number of hiiden layers, number of neurons, learning rate, and input shape. Returns compiled model. Keyword Arguments: n_hidden (integer): Number of hidden layers added to model, excludes input and output layer. Default = 2 n_neuron (integer): Number of neurons to add to each hidden layer. Default = 50 learning_rate (float): Model learning rate that is passed to model optimizer. Smaller values are slower, High values are prone to unstable training. Default = 0.001 in_shape (integer): Input dimension should match number of features. Default = 200 but should be overridden. drop (float): Dropout probability. 1 means drop everything, 0 means drop nothing. Default = 0. Recommended = 0.2-0.6 """ model = keras.models.Sequential() # use dropout layer as input. model.add(keras.layers.Dropout(drop, input_shape=(in_shape,))) # in_shape should be iterable (tuple) for layer in range(n_hidden): # create hidden layers model.add(keras.layers.Dense(n_neuron, activation="relu")) model.add(keras.layers.Dropout(drop)) # add dropout to model after the a dense layer model.add(keras.layers.Dense(1)) # output layer optimizer = keras.optimizers.Adam(learning_rate=learning_rate) model.compile(loss="mse", optimizer=optimizer, metrics=[RootMeanSquaredError(name='rmse')]) return model
def _build_model(self) -> Sequential: model = Sequential() for state in self.sequence: if isinstance(state, Input): model.add(layers.Input(shape=(state.units,))) elif isinstance(state, Hidden): model.add(layers.Dense(units=state.units, activation=state.activation)) elif isinstance(state, Dropout): model.add(layers.Dropout(rate=state.rate)) elif isinstance(state, Output): model.add(layers.Dense(units=state.units, activation=state.activation)) optimizer = OPTIMIZER if optimizer == 'adam': optimizer = Adam(LEARNING_RATE) elif optimizer == 'nadam': optimizer = Nadam(LEARNING_RATE) elif optimizer == 'rmsprop': optimizer = RMSprop(LEARNING_RATE) metrics = [] for metric in METRICS: if metric == 'r_square': metric = RSquare(name=metric, y_shape=(1,)) elif metric == 'rmse': metric = RootMeanSquaredError(metric) metrics.append(metric) model.compile( optimizer=optimizer, loss=LOSS, metrics=metrics ) return model
def build_model(self): height, width, n_channels = self.img_size, self.img_size, self.num_variables model = Sequential([ Conv2D(filters=32, kernel_size=3, padding='same', input_shape=(height, width, n_channels)), BatchNormalization(), Activation('relu'), MaxPooling2D(pool_size=2), Conv2D(filters=32, kernel_size=3, padding='same'), BatchNormalization(), Activation('relu'), MaxPooling2D(pool_size=2), Dropout(0.25), Flatten(), Dense(256, activation='relu'), Dropout(0.25), Dense(self.num_variables, activation='linear') ]) model.compile(loss=self.loss, optimizer=self.optimizer, metrics=[RootMeanSquaredError()]) return model
def build_MLP(optimizer="adam", learning_rate=1e-4, loss="mse"): """Build, compile and return an MLP model of with the given params Parameters ---------- optimizer : str, optional {'adam', 'rmsprop'} The Keras optimizer you would like to use (case insensitive input), by default 'adam' learning_rate : float, optional The learning rate, by default 1e-4 loss : str, optional {all keras losses are accepted} Keras loss to use, by default 'mse' Returns ------- model MLP sequential Keras model compiled with given params """ model = Sequential( [ Dense(500, activation="relu"), Dense(250, activation="relu"), Dense(125, activation="relu"), Dense(62, activation="relu"), Dense(30, activation="relu"), Dense(15, activation="relu"), Dense(7, activation="relu"), Dense(1), ] ) optimizer_object = get_optimizer(optimizer=optimizer, learning_rate=learning_rate) model.compile( loss=loss, optimizer=optimizer_object, metrics=[RootMeanSquaredError()] ) return model
def create_model(args, learning_rate, l1): hidden_layers = [int(n) for n in args.hidden_layers.split(',')] inputs = Input(shape=[N_FEATURES]) hidden = inputs if hidden_layers != [-1]: for size in hidden_layers: hidden = Dense(size, kernel_regularizer=L1L2(l1=l1), bias_regularizer=L1L2(l1=l1))(hidden) hidden = BatchNormalization()(hidden) hidden = ReLU()(hidden) outputs = Dense(1)(hidden) model = Model(inputs=inputs, outputs=outputs) # I know this is ugly, but I added the sgd arg only later so older networks # do not have args.optimizer (and were optimized with Adam) try: if args.optimizer == "sgd": optimizer = SGD(learning_rate=learning_rate, momentum=0.99, nesterov=True) elif args.optimizer == "adam": optimizer = Adam(learning_rate=learning_rate) except AttributeError: optimizer = Adam(learning_rate=learning_rate) model.compile( optimizer=optimizer, loss='mse', metrics=[RootMeanSquaredError(), MeanAbsoluteError(), RSquare()]) return model
def __init__( self, user_size=100, item_size=100, representation_layers=[], embedding_size=16, matching_layers=[32], activation="relu", ): def joinLst(x): return "_".join([str(_) for _ in x]) self.backup_path = f"./training/deepcf__{joinLst(representation_layers)}__{joinLst([embedding_size]+matching_layers)}/mdl.ckpt" self.cp_callback = ModelCheckpoint( filepath=self.backup_path, save_weights_only=True, verbose=0 ) inputs = self._create_inputs(user_size, item_size) representation_model = self._create_representation_model( inputs, representation_layers, activation ) matchingfunction_model = self._create_matchingfunction_model( inputs, embedding_size, matching_layers, activation ) fusion_layer = Concatenate()([representation_model, matchingfunction_model]) output = Dense(1, activation="sigmoid")(fusion_layer) self.model = Model(inputs, output, name="DeepCF") self.model.compile( optimizer="adam", loss=BinaryCrossentropy(), metrics=[RootMeanSquaredError()], )
def __init__(self, size1=512, size2=128, gru_length=20): self.backup_path = f"./training/zeroshot__{size1}__{size2}/mdl.ckpt" self.cp_callback = ModelCheckpoint( filepath=self.backup_path, save_weights_only=True, verbose=0 ) user_input = Input(shape=(gru_length, 768)) item_input = Input(shape=(768)) self.inputs = [user_input, item_input] layer1 = Dense(size1, activation="relu") layer2 = Dense(size2, activation="relu") self.layers = [layer1, layer2] self.gru = GRU(size2) user_present = self.gru(layer2(layer1(user_input))) item_present = layer2(layer1(item_input)) output = Activation(activation="sigmoid")( Dot(axes=1)([user_present, item_present]) ) self.model = Model(self.inputs, output, name="ZeroShot") self.model.compile( optimizer="adam", loss=BinaryCrossentropy(), metrics=[RootMeanSquaredError()], ) self._update_models() self._gen_score_layer(size2)
def _model_fit(train, config): # prepare data train_data = _series_to_supervised(train, n_in=config.n_input) X_train, y_train = train_data[:, :-1], train_data[:, -1] # define model model = Sequential() model.add( Dense(config.n_nodes, activation=config.activation, input_dim=config.n_input) ) model.add(Dense(1)) # compile model.compile( loss=config.loss, optimizer=config.optimizer, metrics=[RootMeanSquaredError()] ) # fit history = model.fit( X_train, y_train, epochs=config.n_epochs, batch_size=config.n_batch, verbose=config.verbose, shuffle=False, validation_split=config.val_split, callbacks=[WandbCallback()], ) return (model, history)
def modelCompile(self, model, learning_rate, decay, amsgrad): #compile the model adam = Adam(learning_rate=learning_rate, decay=decay, amsgrad=amsgrad) model.compile(loss='mean_squared_error', optimizer=adam, metrics=[RootMeanSquaredError(), 'mae', 'acc']) return model
def build_LSTM(config): # Add (config.num_layers - 1) layers that return sequences lstm_list = [ LSTM( config.num_nodes, return_sequences=True, stateful=True, batch_input_shape=(config.n_batch, config.n_input, 1), dropout=config.dropout, recurrent_dropout=config.recurrent_dropout, ) for _ in range(config.num_layers - 1) ] # Final layer does not return sequences lstm_list.append( LSTM( config.num_nodes, return_sequences=False, stateful=True, batch_input_shape=(config.n_batch, config.n_input, 1), dropout=config.dropout, recurrent_dropout=config.recurrent_dropout, ) ) # Single node output layer lstm_list.append(Dense(1)) model = Sequential(lstm_list) optimizer = get_optimizer(config) model.compile( loss=config.loss, optimizer=optimizer, metrics=[RootMeanSquaredError()] ) return model
def iris_model(x_train, y_train, x_val, y_val, params): model = Sequential() model.add( Dense(params['first_neuron'], input_dim=4, activation=params['activation'])) talos.utils.hidden_layers(model, params, 3) model.add(Dense(3, activation='softmax')) if isinstance(params['optimizer'], str): opt = params['optimizer'] else: opt = params['optimizer']() model.compile(optimizer=opt, loss=params['losses'], metrics=['acc', RootMeanSquaredError()]) out = model.fit(x_train, y_train, batch_size=25, epochs=params['epochs'], validation_data=[x_val, y_val], verbose=0) return out, model
def measure_rmse_tf(y_true, y_pred): """Calculate the RMSE score between y_true and y_pred and return it. Parameters ---------- y_true : np.ndarray Array of true values y_pred : np.ndarray Array of predicted values Returns ------- rmse : float The RMSE between the arrays y_true and y_pred """ m = RootMeanSquaredError() m.update_state(y_true, y_pred) rmse = m.result().numpy() return rmse
def train_spatial_temporal_model( model, dataset_generator, opt='adam', epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, include_tb=False): # validation_data, val_steps = VALIDATION_STEPS, ## Early stopping earlystopping = EarlyStopping(monitor='loss', min_delta=0.00001, patience=10, restore_best_weights=True) # val_loss # Automatically save latest best model to file filepath = repo_path + "models/model_saves/" + PRED_TAR + '/' + RUN_ID + ".hdf5" checkpoint = ModelCheckpoint(filepath=filepath, monitor='loss', verbose=0, save_best_only=True, mode='min') # Set callbacks callbacks_list = [checkpoint, earlystopping] # Include tensorboard if include_tb: tensorboard_cb = tf.keras.callbacks.TensorBoard(get_run_logdir()) callbacks_list.extend([tensorboard_cb]) # Optimizers optimizers = { 'adam': Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False) } model.compile(loss='mean_absolute_error', optimizer=optimizers[opt], metrics=[mae, RootMeanSquaredError(), Huber()]) # Fit model #x = [spatial_train, temporal_train_x], y = temporal_train_y, history = model.fit( dataset_generator, epochs=epochs, use_multiprocessing=True, # validation_data = validation_data, validation_steps = val_steps, steps_per_epoch=steps_per_epoch, verbose=1, callbacks=callbacks_list) return (history)
def from_saved(cls, folder): with open(os.path.join(folder, "args.pickle"), "rb") as f: args = argparse.Namespace(**pickle.load(f)) # loads dict and converts it to namespace with open(os.path.join(folder,'model.json')) as f: json_string = json.load(f) model = tf.keras.models.model_from_json(json_string, custom_objects=None) model.load_weights(os.path.join(folder, 'weights.h5')) model.compile( loss ='mse', metrics = [RootMeanSquaredError(), MeanAbsoluteError(), RSquare()] ) return cls(model, args)
def __init__(self, n_input_steps, n_features, n_output_steps, reg=False, drop=False): super().__init__() self.model.add(LSTM(100, input_shape=(n_input_steps, n_features))) self.add_regulization(reg, drop) self.model.add(Dense(n_output_steps)) self.model.compile(optimizer='adam', loss='mse', metrics=[RootMeanSquaredError()])
def create_ensemble(models): if len(models) == 1: return models[0] else: inputs = Input(shape=[N_FEATURES]) predictions = [model(inputs) for model in models] outputs = average(predictions) model = Model(inputs=inputs, outputs=outputs) model.compile( loss='mse', metrics=[RootMeanSquaredError(), MeanAbsoluteError(), RSquare()]) return model
def train(self): """ Train the optimizer. """ self.config.logger.info("DnnOptimiser::train") training_generator = FluctuationDataGenerator( self.config.partition['train'], dirinput=self.config.dirinput_train, **self.config.params) validation_generator = FluctuationDataGenerator( self.config.partition['validation'], dirinput=self.config.dirinput_validation, **self.config.params) model = u_net((self.config.grid_phi, self.config.grid_r, self.config.grid_z, self.config.dim_input), depth=self.config.depth, batchnorm=self.config.batch_normalization, pool_type=self.config.pool_type, start_channels=self.config.filters, dropout=self.config.dropout) if self.config.metrics == "root_mean_squared_error": metrics = RootMeanSquaredError() else: metrics = self.config.metrics model.compile(loss=self.config.lossfun, optimizer=Adam(lr=self.config.adamlr), metrics=[metrics]) # Mean squared error model.summary() plot_model(model, to_file='%s/model_%s_nEv%d.png' % \ (self.config.dirplots, self.config.suffix, self.config.train_events), show_shapes=True, show_layer_names=True) #log_dir = "logs/" + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") log_dir = 'logs/' + '%s_nEv%d' % (self.config.suffix, self.config.train_events) tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1) model._get_distribution_strategy = lambda: None his = model.fit(training_generator, validation_data=validation_generator, use_multiprocessing=False, epochs=self.config.epochs, callbacks=[tensorboard_callback]) self.__plot_train(his) self.save_model(model)
def ModelNN(): classifier = Sequential([ #Dense(25, activation = 'relu', input_dim=5), #BatchNormalization(), LSTM(64, input_shape=(8, 1), activation='tanh', return_sequences=True), Conv1D(32, 8, activation='tanh', strides=2, padding='same'), AveragePooling1D(pool_size=3, strides=1, padding='same'), Dense(12, activation='relu'), LeakyReLU(alpha=0.01), Dense(1), #Dropout(0.2) ]) classifier.compile(optimizer='adam', loss='huber', metrics=[RootMeanSquaredError(), 'mean_squared_error']) return classifier
def createNN(neurons=200, dropOutRate=0.3): ann = Sequential() ann.add( Dense(units=neurons, activation='relu', input_dim=M.shape[1], kernel_initializer=GlorotNormal())) ann.add(Dropout(dropOutRate)) ann.add(Dense(units=int(neurons / 2), activation='relu')) ann.add(Dropout(dropOutRate)) ann.add(Dense(units=int(neurons / 4), activation='relu')) ann.add(Dropout(dropOutRate)) ann.add(Dense(units=1, activation='linear')) ann.compile(optimizer='adam', loss='mse', metrics=['accuracy', RootMeanSquaredError()]) return ann
def create_model(n_feature): ip_layer = Input(shape=(100, n_feature)) #ip_layer = Input(shape = (100,54))(input_shape) lstm_layer1 = (LSTM(50, return_sequences=True, input_shape=(100, n_feature), kernel_regularizer=reg))(ip_layer) lstm_layer2 = (LSTM(50, return_sequences=True, kernel_regularizer=reg))(lstm_layer1) lstm_layer3 = (LSTM(50, return_sequences=True, kernel_regularizer=reg))(lstm_layer2) lstm_layer4 = (LSTM(50, return_sequences=True, kernel_regularizer=reg))(lstm_layer3) output_layer = TimeDistributed(Dense(1, activation="linear"))(lstm_layer4) model = tf.keras.Model(inputs=ip_layer, outputs=output_layer) model.compile(optimizer='adam', loss=rmse_loss, metrics=RootMeanSquaredError()) return model
def __init__(self, hparams, name, log_dir): self.univariate = hparams.get('UNIVARIATE', True) self.batch_size = int(hparams.get('BATCH_SIZE', 32)) self.epochs = int(hparams.get('EPOCHS', 500)) self.patience = int(hparams.get('PATIENCE', 15)) self.val_frac = hparams.get('VAL_FRAC', 0.15) self.T_x = int(hparams.get('T_X', 32)) self.metrics = [ MeanSquaredError(name='mse'), RootMeanSquaredError(name='rmse'), MeanAbsoluteError(name='mae'), MeanAbsolutePercentageError(name='mape') ] self.standard_scaler = StandardScaler() self.forecast_start = datetime.datetime.today() model = None super(NNModel, self).__init__(model, self.univariate, name, log_dir=log_dir)
def build_LSTM_small(config): model = Sequential( [ LSTM( 100, return_sequences=True, stateful=True, batch_input_shape=(config.n_batch, config.n_input, 1), ), LSTM(50, return_sequences=True, stateful=True), LSTM(25, return_sequences=True, stateful=True), LSTM(12, return_sequences=True, stateful=True), LSTM(7, stateful=True), Dense(1), ] ) optimizer = get_optimizer(config) model.compile( loss=config.loss, optimizer=optimizer, metrics=[RootMeanSquaredError()] ) return model
def __init__(self, n_input_steps, n_features, n_output_steps, reg=False, drop=False, n_LSTM_hidden_layers=1, n_cells=100): super().__init__() self.model.add( LSTM(n_cells, return_sequences=True, input_shape=(n_input_steps, n_features))) for _ in range(n_LSTM_hidden_layers): self.model.add(LSTM(n_cells, return_sequences=True)) self.model.add(LSTM(n_cells)) self.add_regulization(reg, drop) self.model.add(Dense(n_output_steps)) self.model.compile(optimizer='adam', loss='mse', metrics=[RootMeanSquaredError()])
def linear_model(): model = Sequential() model.add( Dense(math.ceil(1.2 * features / 2), kernel_initializer='random_normal', activation='relu', input_dim=features)) model.add(Dropout(0.4)) model.add( Dense(math.ceil(0.6 * features), kernel_initializer='random_normal', activation='relu', kernel_constraint=maxnorm(5))) model.add(Dropout(0.2)) model.add( Dense(1, kernel_initializer='normal', kernel_constraint=maxnorm(5))) model.compile(optimizer=Adam(lr=0.0001), loss=MeanSquaredError(), metrics=[RootMeanSquaredError()]) return model
def build_model(self): input_layer = Input((self.ts_length, self.num_variables)) x = input_layer input_res = input_layer for d in range(self.depth): x = self._inception_module(x) if self.use_residual and d % 3 == 2: x = self._shortcut_layer(input_res, x) input_res = x gap_layer = GlobalAveragePooling1D()(x) output_layer = Dense(self.num_variables, activation='linear')(gap_layer) model = Model(inputs=input_layer, outputs=output_layer) model.compile(loss=self.loss, optimizer=self.optimizer, metrics=[RootMeanSquaredError()]) return model
def build_MLP(config): # Do we need to put input_dim=config.n_input in first layer? # dense_list = [Dense(config.n_nodes, activation=config.activation) for _ in range(config.num_layers)] # dense_list.append(Dense(1)) # model = Sequential(dense_list) model = Sequential( [ Dense(500, activation="relu"), Dense(250, activation="relu"), Dense(125, activation="relu"), Dense(62, activation="relu"), Dense(30, activation="relu"), Dense(15, activation="relu"), Dense(7, activation="relu"), Dense(1), ] ) optimizer = get_optimizer(config) model.compile( loss=config.loss, optimizer=optimizer, metrics=[RootMeanSquaredError()] ) return model
def __init__(self, n_input_steps, n_features, n_output_steps, reg=False, drop=False, n_LSTM_hidden_layers=1, n_cells=200): self.model = Sequential() self.model.add( ConvLSTM2D(64, (1, 3), activation='relu', input_shape=(1, 1, n_input_steps, n_features))) self.model.add(Flatten()) self.model.add(RepeatVector(n_output_steps)) for _ in range(n_LSTM_hidden_layers): self.model.add( LSTM(n_cells, activation='relu', return_sequences=True)) self.add_regulization(reg, drop) self.model.add(TimeDistributed(Dense(1))) self.model.compile(optimizer='adam', loss='mse', metrics=[RootMeanSquaredError()])