def run_reg(self):
self.train_x, _, self.train_y = self.preprocess(
self.training,
0.,
time_step=self.time_step,
num_feature=14,
normalize=False)
if self.pca_flag:
self.train_x, self.test_x = self.pca(
self.train_x, self.test_x, n_components=self.n_components)
def min_max_scale(X, range=(0, 1)):
mi, ma = range
self.min = X.min()
self.max = X.max()
X_std = (X - X.min()) / (X.max() - X.min())
X_scaled = X_std * (ma - mi) + mi
return X_scaled
self.train_x = min_max_scale(self.train_x)
self.nn = KerasRegressor(build_fn=self.create_reg_model,
epochs=self.epoch,
batch_size=self.batch,
validation_split=0.05,
shuffle=True,
verbose=2)
self.nn.fit(self.train_x, self.train_y['Y_M_1'])
def keras_grid_search(X,Y,model_fn=create_model):
# kears_estimator = KerasClassifier(build_fn=model_fn, verbose=1)
kears_estimator = KerasRegressor(build_fn=create_model, verbose=1)
optimizers = ['adam','sgd']#, 'Adamax', 'Nadam']
epochs = [30, 50, 100]
hl1_nodes = [10,20, 50]
btcsz = [32, 64, 128]
loss=[rmse,'mean_squared_error']
activation=['relu','tanh','sigmoid']
# learning_rate= [0.001, 0.01, 0.0001]
param_grid = dict(optimizer=optimizers, hl1_nodes=hl1_nodes,
nb_epoch=epochs, batch_size=btcsz,activation=activation,loss=loss)
grid = GridSearchCV(estimator=kears_estimator, param_grid=param_grid,
scoring='neg_mean_absolute_error', n_jobs=-1,
verbose=3)
grid_result = grid.fit(X, Y)
# Show the results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
def _grid_optimization(self):
X = self.data.train_set.get_input(self.model_name)
Y = self.data.train_set.get_output()
build_fn = self.model_builder.build_model
possible_params = self.model_builder.possible_parameters
regressor = KerasRegressor(build_fn=build_fn, verbose=1)
grid = GridSearchCV(estimator=regressor,
param_grid=possible_params,
n_jobs=1,
cv=5)
grid_result = grid.fit(X, Y)
# summarize results
print(
f"Best: {-grid_result.best_score_} using {grid_result.best_params_}"
)
means = grid_result.cv_results_["mean_test_score"]
stds = grid_result.cv_results_["std_test_score"]
params = grid_result.cv_results_["params"]
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (-mean, stdev, param))
model = grid_result.best_estimator_.model
mape = "mean_absolute_percentage_error"
model.compile(loss=mape, optimizer="adam", metrics=[mape])
return model
def build(self, **kwargs):
"""
Builds and returns estimator.
Args:
kwargs (key-value(int)): The user must specify ``input_dim`` and ``num_samples``.
Returns:
`sklearn pipeline` object: pipeline for transforming the features and training the estimator
"""
if 'input_dim' not in kwargs:
raise ValueError('You need to specify input dimensions when building the model.')
if 'num_samples' not in kwargs:
raise ValueError('You need to specify num_samples when building the keras model.')
if 'num_classes' not in kwargs:
raise ValueError('You need to specify num_classes when building the keras model.')
input_dim=kwargs['input_dim']
num_samples = kwargs['num_samples']
num_classes = kwargs['num_classes']
#the arguments of ``build_fn`` are not passed directly. Instead they should be passed as arguments to ``KerasClassifier``.
estimator = KerasRegressor(build_fn=self._keras_model,
input_dim=input_dim)
grid = GridSearchCV(estimator=estimator,
param_grid=self._hyperparameters,
cv=self._num_cv_folds,
refit=self._refit,
verbose=self._verbose)
return self._create_pipeline(estimator=estimator)
def random_grid_search(X,Y,model_fn=create_model):
kears_estimator = KerasRegressor(build_fn=create_model, verbose=1)
optimizers = ['adam'] # , 'Adamax', 'Nadam']
epochs = [30, 50, 100]
hl1_nodes = [10, 20, 50]
btcsz = [32, 64, 128]
loss = [rmse, 'mean_squared_error']
activation = ['relu', 'tanh', 'sigmoid']
# learning_rate= [0.001, 0.01, 0.0001]
param_grid = dict(optimizer=optimizers, hl1_nodes=hl1_nodes,
nb_epoch=epochs, batch_size=btcsz, activation=activation, loss=loss)
n_iter_search = 80 # Number of parameter settings that are sampled.
random_search = RandomizedSearchCV(estimator=kears_estimator,
param_distributions=param_grid,
n_iter=n_iter_search,scoring='neg_mean_absolute_error', n_jobs=-1,
verbose=3)
random_search.fit(X, Y)
# Show the results
print("Best: %f using %s" % (random_search.best_score_, random_search.best_params_))
means = random_search.cv_results_['mean_test_score']
stds = random_search.cv_results_['std_test_score']
params = random_search.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
def evaluate_gru(row, x, y, validation_size):
"""
Evaluate the model with the params given in the row
:param row: The row to get the parameters from
:param x: The x input vector
:param y: The y target vector
:param validation_size: The validation distance to use for scoring
:return:
"""
from tensorflow.python.keras.wrappers.scikit_learn import KerasRegressor
scaled_x, scaled_y, x_scaler, y_scaler = scale_inputs(x, y)
scaled_x = scaled_x.values.reshape(scaled_x.shape[0], scaled_x.shape[1], 1)
scaled_y = scaled_y.values
used_x = scaled_x[:-validation_size]
used_y = scaled_y[:-validation_size]
params = get_gru_params(row)
times = dict()
start = time.clock()
model = KerasRegressor(build_fn=create_gru, **params)
fit = model.fit(used_x, used_y, verbose=0)
times['fit_time'] = time.clock() - start
start = time.clock()
prediction = rescale_array(model.predict(scaled_x[-validation_size:]),
y_scaler)
norm_factor = 1 / (y.max() - y.min())
times['prediction_time'] = time.clock() - start
scores = score_prediction(y[-validation_size:], prediction, norm_factor)
return {
'params': params,
'prediction': prediction,
**times,
**scores, 'norm_factor': norm_factor
}
def grid_search():
"""HyperParameter Optimization"""
print('start grid search')
batch_size_grid = [20, 50, 100]
epochs_grid = [50, 150]
optimizer_grid = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']
#learn_rate_grid = [0.001, 0.01, 0.1, 0.2, 0.3]
#init_mode_grid = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']
#activation_grid = ['relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']
#first_neurons_grid = [50, 100]
train = np_training_dataframe
target = np_training_target
param_grid = dict(batch_size=batch_size_grid)
model = KerasRegressor(build_fn=create_model(inputtrain=train))
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=1, cv=2, verbose=2, scoring='neg_mean_squared_error')
grid_result = grid.fit(X=train, y=target)
print('Reached Here')
def grids(self, time_steps):
# LSTM Input Shape
time_steps = 1 # number of time-steps you are feeding a sequence (?)
inputs_numb = self.X_train.shape[1] # number of inputs
input_shape = [time_steps, inputs_numb]
# ,input_shape=input_shape
model = KerasRegressor(build_fn=create_model, verbose=1)
# GridSearch code
start = time()
optimizers = ['adam', 'Adamax', 'Nadam']
epochs = [30, 50, 100]
hl1_nodes = [10, 20, 50]
btcsz = [32, 64, 128]
# learning_rate= [0.001, 0.01, 0.0001]
param_grid = dict(optimizer=optimizers,
hl1_nodes=hl1_nodes,
nb_epoch=epochs,
batch_size=btcsz)
scoring = make_scorer(
accuracy_score) # in order to use a metric as a scorer
# scoring = estimator.score(self.X_test,self.Y_test)
grid = GridSearchCV(estimator=model,
param_grid=param_grid,
scoring='neg_mean_absolute_error',
n_jobs=-1,
verbose=3)
grid_result = grid.fit(self.X_train, self.Y_train)
print("Best: %f using %s" %
(grid_result.best_score_, grid_result.best_params_))
# for params, mean_score, scores in grid_result.cv_results_:
# print("%f (%f) with: %r" % (scores.mean(), scores.std(), params))
for x in grid_result.cv_results_:
print(x)
print("total time:", time() - start)
def estimate_gru(x, y, batch_size):
"""
Estimates best parameters for GRU given input samples and targets.
Estimated parameters are: weights, dropout_rate, epochs, batch_size and learning rate.
:param x: The samples dataframe
:param y: The targets series
:param batch_size: The batch size for the gru
"""
from tensorflow.python.keras.wrappers.scikit_learn import KerasRegressor
from tensorflow.python.keras.backend import clear_session
clear_session()
logger.log('Finding best GRU parameters', 3)
x, y, x_scaler, y_scaler = scale_inputs(x, y)
x = x.values.reshape(x.shape[0], x.shape[1], 1)
y = y.values
# batch_size = [24, 24 * 7] if rate == 'H' else [7, 7 * 30]
gru = RandomizedSearchCV(KerasRegressor(create_gru, verbose=0),
param_distributions={
'weights': np.linspace(1, 100, 20, endpoint=True, dtype=int),
'dropout_rate': np.linspace(0.1, 0.3, 3, endpoint=True),
'input_shape': [(x.shape[1], x.shape[2])],
'epochs': range(1, 10 + 1),
'batch_size': [batch_size],
'learning_rate': np.linspace(0.001, 0.02, 10, endpoint=True)
},
cv=TimeSeriesSplit(),
scoring=lambda estimator, X, y, **kwargs: tensorflow_score(estimator, X, y, y_scaler,
batch_size, **kwargs),
verbose=2,
n_jobs=-1,
n_iter=20)
gru.fit(x, y)
logger.log(f'Found best GRU model with params {gru.best_params_} and score {gru.best_score_}', 3)
return gru.best_params_, gru.best_score_
def optimize_hyperparameter(X, Y):
#Define hypermeters values for optimization
model = KerasRegressor(build_fn=build_DNN, verbose=0)
batch_size = [5, 25, 50] #Add more later
epochs = [250] #Add more later
optimizer = ['RMSprop', 'Nadam'] #Reduce later
drop_out = [0.1, 0.2, 0.3, 0.4, 0.5]
activation = ['relu', 'sigmoid'] #reduce later
neurons1 = [128 * 1, 128 * 2, 128 * 3, 128 * 4]
neurons2 = [128 * 1, 128 * 2, 128 * 3, 128 * 4]
param_grid = dict(neurons1=neurons1,
neurons2=neurons2,
optimizer=optimizer,
dropout_rate=drop_out,
activation=activation,
batch_size=batch_size,
epochs=epochs)
scoring = {
'MSE': make_scorer(mean_squared_error, greater_is_better=False),
'MAE': make_scorer(mean_absolute_error, greater_is_better=False),
'EVS': make_scorer(explained_variance_score, greater_is_better=True)
}
#Run gridsearch
grid = GridSearchCV(
estimator=model,
param_grid=param_grid,
cv=3,
scoring=scoring,
refit='MSE',
n_jobs=-1
) #Define grid with listed parameters. This includes inner CV and no. of folds is changed later to 10
best_param_list = [
] #list of best param combinations fromeach round of outer CV
best_pred_list = [] #list of best predictions from each round of outer CV
cv_score_list = [
] #list of cv scores of all models in all rounds of outer CV
best_testset_list = []
epoch_loss_list = []
cv = KFold(n_splits=5, random_state=42,
shuffle=True) #Outer CV and no. of folds is changed later to 10
for train_index, test_index in cv.split(X):
print('New fold!')
X_train, X_test, Y_train, Y_test = X.iloc[train_index, ], X.iloc[
test_index, ], Y.iloc[train_index, ], Y.iloc[test_index, ]
#X_train = pca.fit_transform(X_train)
grid_result = grid.fit(X_train,
Y_train) #Include inner cross-validation
cv_score_list.append(grid_result.cv_results_)
if grid_result.best_params_ not in best_param_list:
best_param_list.append(grid_result.best_params_)
print('New best param combi is added: ',
grid_result.best_params_) #For checking
#Predict X_test using newly obtained best model
optimized_model = grid_result.best_estimator_ #Build the model using best params
Y_pred = optimized_model.predict(X_test)
best_pred_list.append(Y_pred)
best_testset_list.append(Y_test)
epoch_loss_list.append(
grid_result.best_estimator_.model.history.history)
else: #This is just for checking
print('Current best combi is already there.')
return (best_param_list, best_pred_list, cv_score_list, best_testset_list,
epoch_loss_list)
def buildmodel():
# design network
model = Sequential()
model.add(
LSTM(lstm_units, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1, activation='relu'))
model.compile(loss='mae', optimizer='adam')
return model
buildmodel().summary()
estimator = KerasRegressor(build_fn=buildmodel,
epochs=amount_of_epochs,
batch_size=batch_size,
verbose=2)
if cross_validation:
# evaluation
# initialize the cross validation folds api
kfold = KFold(k_folds)
results = cross_val_score(estimator, test_X, test_y, cv=kfold)
print(results)
if plot:
# fit network
history = buildmodel().fit(train_X,
train_y,
batch_size=batch_size,
epochs=amount_of_epochs,
class NeuralNet(Preprocess):
def __init__(self,
training,
time_step=5,
epoch=10,
pca_flag=False,
batch=256,
n_components=4):
self.training = training
self.time_step = time_step
self.epoch = epoch
self.pca_flag = pca_flag
self.n_components = n_components
self.batch = batch
def create_cls_network(self) -> Sequential:
seq = Sequential()
seq.add(
LSTM(128, input_shape=(self.time_step, 55), return_sequences=True))
seq.add(
LSTM(128,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True))
seq.add(Flatten())
seq.add(Dense(128, activation='relu'))
seq.add(Dense(3, activation='softmax'))
seq.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['categorical_accuracy'])
return seq
def create_reg_model(self, ) -> Sequential:
seq = Sequential()
seq.add(
Dense(units=128,
input_dim=self.train_x.shape[1],
activation='relu'))
seq.add(Dense(units=32, activation='relu'))
seq.add(Dense(units=1))
seq.compile(loss='mean_squared_error', optimizer='adam')
return seq
def run_cls(self):
self.train_x, self.train_y, _ = self.preprocess(
self.training, 0.0015, time_step=self.time_step)
self.nn = self.create_cls_network()
self.nn.fit(self.train_x.values.reshape((-1, self.time_step, 55)),
np_utils.to_categorical(self.train_y['Label_1'],
num_classes=3),
epochs=self.epoch,
validation_split=0.05,
shuffle=False)
def predict_cls(self):
processed_test, y, _ = self.preprocess(self.test,
0.0015,
time_step=self.time_step)
_, test_acc = self.nn.evaluate(
x=processed_test.values.reshape(-1, self.time_step, 55),
y=np_utils.to_categorical(y['Label_1'], num_classes=3),
steps=self.time_step)
# print(f'Train acc - {train_acc}')
print(f'Test acc - {test_acc}')
y_pred = self.nn.predict(x=processed_test.values.reshape(
-1, self.time_step, 55),
steps=self.time_step)
def run_reg(self):
self.train_x, _, self.train_y = self.preprocess(
self.training,
0.,
time_step=self.time_step,
num_feature=14,
normalize=False)
if self.pca_flag:
self.train_x, self.test_x = self.pca(
self.train_x, self.test_x, n_components=self.n_components)
def min_max_scale(X, range=(0, 1)):
mi, ma = range
self.min = X.min()
self.max = X.max()
X_std = (X - X.min()) / (X.max() - X.min())
X_scaled = X_std * (ma - mi) + mi
return X_scaled
self.train_x = min_max_scale(self.train_x)
self.nn = KerasRegressor(build_fn=self.create_reg_model,
epochs=self.epoch,
batch_size=self.batch,
validation_split=0.05,
shuffle=True,
verbose=2)
self.nn.fit(self.train_x, self.train_y['Y_M_1'])
def predict_reg(self, test):
self.test_x, _, self.test_y = self.preprocess(test,
0.,
time_step=self.time_step,
num_feature=14,
normalize=False)
self.test_x = (self.test_x - self.min) / (self.max - self.min)
y_pred = self.nn.predict(self.test_x)
try:
r2 = r2_score(self.test_y.Y_M_1, y_pred.reshape(-1, 1))
except:
r2 = 0
print('Something wrong')
print(f'R-square is {r2}')
# print(f'Mean - y_pred {np.mean(y_pred)}, Mean - y {np.mean(self.test_y.Y_M_1)}')
return r2
print(x1)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3)
def build_regressor():
regressor = Sequential()
regressor.add(Dense(units=17, input_dim=17))
regressor.add(Dense(units=1))
regressor.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['mae', 'accuracy'])
return regressor
regressor = KerasRegressor(build_fn=build_regressor, batch_size=32, epochs=15)
results = regressor.fit(X_train, y_train)
y_pred = regressor.predict(X_test)
fig, ax = plt.subplots()
ax.scatter(y_test, y_pred)
ax.plot([y_test.min(), y_test.max()],
[y_test.min(), y_test.max()],
'k--',
lw=4)
ax.set_xlabel('Measured')
ax.set_ylabel('Predicted')
plt.show()
inp = np.array([[74], [60], [45], [67], [49], [43], [33], [45], [57], [29.68],
[10], [7], [2], [0], [20], [4], [31]])