def train(self, train=[]):
X, y, X_cnn, X_lstm = self.load_data()
if y.isna().any().values[0]:
X = X.drop(y.index[np.where(y.isna())[0]])
if len(X_cnn.shape) > 1:
X_cnn = np.delete(X_cnn, np.where(y.isna())[0], axis=0)
if len(X_lstm.shape) > 1:
X_lstm = np.delete(X_lstm, np.where(y.isna())[0], axis=0)
y = y.drop(y.index[np.where(y.isna())[0]])
if self.static_data['type'] == 'pv' and self.static_data[
'NWP_model'] == 'skiron':
index = np.where(X['flux'] > 1e-8)[0]
X = X.iloc[index]
y = y.iloc[index]
X_cnn = X_cnn[index]
X_new = X.copy(deep=True)
if self.static_data['train_online']:
X, y, X_cnn, X_lstm = self.merge_old_data(X,
y,
X_cnn=X_cnn,
X_lstm=X_lstm)
if self.static_data['type'] == 'pv' and self.static_data[
'NWP_model'] == 'skiron':
index = np.where(X['flux'] > 1e-8)[0]
X = X.iloc[index]
y = y.iloc[index]
X_cnn = X_cnn[index]
X1 = self.scale(X)
self.scale_y = MinMaxScaler(feature_range=(.1, 20)).fit(y.values)
X_new = pd.DataFrame(self.sc.transform(X_new.values),
columns=X_new.columns,
index=X_new.index)
y1 = pd.DataFrame(self.scale_y.transform(y.values),
columns=y.columns,
index=y.index)
if not self.static_data['clustering'][
'is_clustering_trained'] and not os.path.exists(
os.path.join(
self.static_data['path_fuzzy_models'],
self.static_data['clustering']['cluster_file'])):
self.train_fuzzy_clustering(X1, y1)
self.clusterer = clusterer(
self.static_data['path_fuzzy_models'],
self.static_data['clustering']['cluster_file'],
self.static_data['type'])
self.logger.info('Clusters created')
def predict_individual(self, X, y=None, X_lstm=None):
self.load()
self.clusterer=clusterer(self.static_data['path_fuzzy_models'],self.static_data['clustering']['cluster_file'],self.static_data['type'])
self.clusterer.activations = self.clusterer.compute_activations(X)
if isinstance(self.clusterer.activations, pd.Series):
self.clusterer.activations = pd.DataFrame(self.clusterer.activations.to_dict(), index=[0])
predictions, methods = self.initialize_predictions(self.clusterer.activations, X.index)
X = pd.DataFrame(self.sc.transform(X.values), columns=X.columns, index=X.index)
if len(self.var_nonreg)>0:
X=X.drop(columns=self.var_nonreg).copy(deep=True)
train_clust = []
for clust in self.clusterer.activations.columns:
indices = self.clusterer.activations[clust].index[self.clusterer.activations[clust] >= self.thres_act]
inputs = X.loc[self.clusterer.activations[clust] >= self.thres_act]
if not inputs.shape[0] == 0:
train_clust.append(clust)
for clust in train_clust:
indices = self.clusterer.activations[clust].index[
self.clusterer.activations[clust] > self.thres_act]
inputs = X.loc[self.clusterer.activations[clust] > self.thres_act]
# targets = y.loc[indices]
if not inputs.shape[0] == 0:
cluster_dir = os.path.join(self.static_data['path_project'], 'Regressor_layer/' + clust)
if self.lstm:
if not X_lstm is None:
model = lstm_3d(cluster_dir, self.static_data['rated'])
nind = np.where(self.clusterer.activations[clust] >= self.thres_act)[0]
nind.sort()
pred = model.predict(X_lstm[nind])
predictions['LSTM'].loc[self.clusterer.activations[clust] > self.thres_act, clust]=pred
# np.save(os.path.join(cluster_dir, 'data/X_test.npy'), inputs.values)
# np.save(os.path.join(cluster_dir, 'data/y_test.npy'), targets.values)
static_data = self.static_data
clust_regressor = regressor_layer(static_data, cluster_dir)
pred, regressors = clust_regressor.cluster_predict(inputs.values)
for i in range(len(regressors)):
p=self.scale_y.inverse_transform(pred[:, i].reshape(-1,1))
p[p < 0] = 0
predictions[regressors[i]].loc[self.clusterer.activations[clust] > self.thres_act,clust] = p.ravel()
return predictions
def train_TL(self, path_model_tl, train=[]):
model_tl = self.load_to_transfer(path_model_tl)
static_data_tl = self.static_data['tl_project']['static_data']
self.sc = model_tl['sc']
self.scale_y = model_tl['scale_y']
X, y, X_cnn, X_lstm = self.load_data()
if y.isna().any().values[0]:
X = X.drop(y.index[np.where(y.isna())[0]])
if len(X_cnn.shape) > 1:
X_cnn = np.delete(X_cnn, np.where(y.isna())[0], axis=0)
if len(X_lstm.shape) > 1:
X_lstm = np.delete(X_lstm, np.where(y.isna())[0], axis=0)
y = y.drop(y.index[np.where(y.isna())[0]])
if self.static_data['type'] == 'pv' and self.static_data[
'NWP_model'] == 'skiron':
index = np.where(X['flux'] > 1e-8)[0]
X = X.iloc[index]
y = y.iloc[index]
X_cnn = X_cnn[index]
X_new = X.copy(deep=True)
if self.static_data['train_online']:
X, y, X_cnn, X_lstm = self.merge_old_data(X,
y,
X_cnn=X_cnn,
X_lstm=X_lstm)
if self.static_data['type'] == 'pv':
index = np.where(X['flux'] > 1e-8)[0]
X = X.iloc[index]
y = y.iloc[index]
X_cnn = X_cnn[index]
X1 = self.scale(X)
# Obsolete
# create_nwp_sampler = nwp_sampler(self.static_data)
# if create_nwp_sampler.istrained == False:
# create_nwp_sampler.train(X1, X_cnn, gpu_id=self.static_data['CNN']['gpus'][0])
self.scale_y = MinMaxScaler(feature_range=(.1, 20)).fit(y.values)
X_new = pd.DataFrame(self.sc.transform(X_new.values),
columns=X_new.columns,
index=X_new.index)
y1 = pd.DataFrame(self.scale_y.transform(y.values),
columns=y.columns,
index=y.index)
fuzzy_file = os.path.join(static_data_tl['path_fuzzy_models'],
static_data_tl['clustering']['cluster_file'])
fmodel = joblib.load(fuzzy_file)
joblib.dump(
fmodel,
os.path.join(self.static_data['path_fuzzy_models'],
self.static_data['clustering']['cluster_file']))
self.clusterer = clusterer(
self.static_data['path_fuzzy_models'],
self.static_data['clustering']['cluster_file'],
self.static_data['type'])
self.logger.info('Clusters created')
train_clust_list = self.find_clusters_for_training(X_new, train)
activations = self.clusterer.compute_activations(X1)
self.save_cluster_data(activations, X1, y1, X_cnn, X_lstm,
train_clust_list)
self.save_global_data(activations, X1, y1, X_cnn, X_lstm)
self.regressors = dict()
gpus = np.tile(self.static_data['CNN']['gpus'], len(train_clust_list))
glob_regressor = global_train_tl(self.static_data, self.sc, gpus[0])
if glob_regressor.istrained == False:
self.logger.info('Global regressor is training..')
self.regressors['Global'] = glob_regressor.fit(
rule_model=model_tl['regressors']['Global'])
self.logger.info('Global regressor trained')
else:
self.regressors['Global'] = glob_regressor.to_dict()
with elapsed_timer() as eval_elapsed:
for k, clust in enumerate(train_clust_list):
t = time.process_time()
print('Begin training of ' + clust)
self.logger.info('Begin training of ' + clust)
clust_regressor = cluster_train_tl(self.static_data, clust,
self.sc, gpus[k])
if clust_regressor.istrained == False:
self.regressors[clust] = clust_regressor.fit(
rule_model=model_tl['regressors'][clust])
else:
self.regressors[clust] = clust_regressor.to_dict()
print('time %s' % str(eval_elapsed() / 60))
self.logger.info('time %s', str((eval_elapsed() - t) / 60))
print('finish training of ' + clust)
self.logger.info('finish training of ' + clust)
self.save()
t = eval_elapsed()
self.predict_regressors(X1, y1, X_cnn, X_lstm)
combine_model_ = Combine_train(self.static_data)
self.combine_model = combine_model_.train()
self.istrained = True
self.full_trained = True
self.save()
def train(self, X, y, update_data=True, lstm=False, X_lstm=None):
self.lstm=lstm
X_new=X.copy(deep=True)
y_new = y.copy(deep=True)
if update_data:
X, y = self.save_data(X, y)
X1 = self.scale(X)
self.scale_y = MinMaxScaler(feature_range=(.1, 20)).fit(y.values)
y1 = pd.DataFrame(self.scale_y.transform(y.values), columns=y.columns, index=y.index)
if not self.static_data['clustering']['clustering_trained']: #and not os.path.exists(os.path.join(self.static_data['path_fuzzy_models'],self.static_data['clustering']['cluster_file'])):
N, D = X.shape
n_split = int(np.round(N * 0.85))
X_test = X.iloc[n_split + 1:]
y_test = y1.iloc[n_split + 1:]
X_train = X.iloc[:n_split]
y_train = y1.iloc[:n_split]
optimizer = cluster_optimize(self.static_data['type'], self.var_imp, self.var_lin, self.static_data['path_fuzzy_models'],self.static_data['clustering']['cluster_file'],self.static_data['resampling'],self.static_data['RBF']['njobs'])
optimizer.run(X_train, y_train, X_test, y_test, self.static_data['sklearn']['njobs'])
self.clusterer=clusterer(self.static_data['path_fuzzy_models'],self.static_data['clustering']['cluster_file'],self.static_data['type'])
self.logger.info('Clusters created')
act_new = self.clusterer.compute_activations(X_new)
if len(self.var_nonreg)>0:
X_new=X_new.drop(columns=self.var_nonreg).copy(deep=True)
self.save()
train_clust=[]
for clust in act_new.columns:
indices = act_new[clust].index[act_new[clust] >= self.thres_act].tolist()
if len(indices)>0:
inputs=X_new.loc[act_new[clust] >= self.thres_act]
targets=y_new.loc[act_new[clust] >= self.thres_act]
cluster_dir = os.path.join(self.static_data['path_project'], 'Regressor_layer/' + clust)
if not os.path.exists(os.path.join(cluster_dir, 'data')):
os.makedirs((os.path.join(cluster_dir, 'data')))
np.save(os.path.join(cluster_dir, 'data/X_train.npy'), inputs.values)
np.save(os.path.join(cluster_dir, 'data/y_train.npy'), targets.values)
if not inputs.shape[0]==0:
train_clust.append(clust)
t=0
self.clusterer.activations = self.clusterer.compute_activations(X)
if len(self.var_nonreg)>0:
X1=X1.drop(columns=self.var_nonreg).copy(deep=True)
# train_clust=train_clust[7:]
with elapsed_timer() as eval_elapsed:
for clust in train_clust:
# t = time.process_time()
print('Begin training of ' + clust)
indices = np.where(self.clusterer.activations[clust] >= self.thres_act)[0]
act=self.clusterer.activations.loc[self.clusterer.activations[clust] >= self.thres_act, clust]
inputs=X1.loc[self.clusterer.activations[clust] >= self.thres_act]
targets=y1.loc[self.clusterer.activations[clust] >= self.thres_act]
cluster_dir = os.path.join(self.static_data['path_project'], 'Regressor_layer/' + clust)
if not os.path.exists(os.path.join(cluster_dir, 'data')):
os.makedirs((os.path.join(cluster_dir, 'data')))
np.save(os.path.join(cluster_dir, 'data/X_train.npy'), inputs.values)
np.save(os.path.join(cluster_dir, 'data/y_train.npy'), targets.values)
targets_norm=y.loc[self.clusterer.activations[clust] >= self.thres_act]
if len(targets.shape)==1:
ind=targets.index[pd.isnull(targets).nonzero()[0]]
if ind.shape[0]!=0:
inputs = inputs.drop(ind)
targets = targets.drop(ind)
else:
ind=targets.index[pd.isnull(targets).any(1).nonzero()[0]]
if ind.shape[0] != 0:
inputs = inputs.drop(ind)
targets = targets.drop(ind)
cluster_dir = os.path.join(self.static_data['path_project'], 'Regressor_layer/' + clust)
clust_regressor = regressor_layer(self.static_data, cluster_dir)
clust_regressor.train_cluster(inputs.values,targets.values,act.values)
if hasattr(clust_regressor, 'trained'):
if (clust_regressor.trained != 'ok'):
if self.lstm:
if not X_lstm is None:
model = lstm_3d(cluster_dir, self.static_data['rated'])
nind = np.where(self.clusterer.activations[clust] >= self.thres_act)[0]
nind.sort()
model.train(X_lstm[nind], y.values[nind])
else:
if self.lstm:
if not X_lstm is None:
model = lstm_3d(cluster_dir, self.static_data['rated'])
nind = np.where(self.clusterer.activations[clust] >= self.thres_act)[0]
nind.sort()
model.train(X_lstm[nind], y.values[nind])
clust_regressor.trained = 'ok'
clust_regressor.save(clust_regressor.cluster_dir)
self.save()
print('finish training of ' + clust)
print('time %s' % str(eval_elapsed() / 60))
self.logger.info('time %s', str((eval_elapsed() - t) / 60))
print('finish training of ' + clust)
t=eval_elapsed()
predictions = self.predict_individual(X, y, X_lstm=X_lstm)
combine_dir = os.path.join(self.static_data['path_project'], 'Combine_layer')
if not os.path.exists(combine_dir):
os.makedirs(combine_dir)
# joblib.dump(predictions, os.path.join(combine_dir, 'pred_train_individual.pickle'))
for comb_method in self.static_data['combine_methods']:
comb_method_dir = os.path.join(combine_dir, comb_method)
if not os.path.exists(comb_method_dir):
os.makedirs(comb_method_dir)
if comb_method not in {'cnn'}:
if os.path.exists(os.path.join(comb_method_dir, 'combine_models.pickle')):
combine_models = joblib.load(os.path.join(comb_method_dir, 'combine_models.pickle'))
else:
combine_models = dict()
for rule in train_clust:
x_train = np.array([])
for method in predictions.keys():
if method not in {'activations'}:
p=self.scale_y.transform(predictions[method][rule].values.reshape(-1,1)).ravel()
if x_train.shape[0] == 0:
x_train = p
else:
x_train = np.vstack((x_train, p))
methods = [method for method in predictions.keys() if method not in {'activations'}]
x_train = x_train.T
try:
mask = np.any(~np.isnan(x_train), axis=1)
x_train = x_train[mask, :]
y_train1 = y1.values[mask]
comb = combine_model(comb_method, methods, 24, '', self.rated, comb_method_dir)
comb.train(x_train, y_train1)
combine_models[rule] = comb
except:
continue
joblib.dump(combine_models, os.path.join(comb_method_dir, 'combine_models.pickle'))
else:
X_train3d = np.array([])
for method in predictions.keys():
if method in {'activations'}:
continue
else:
x = predictions[method].copy()
if self.rated is None:
x[x.isnull()] = -999
else:
x[x.isnull()] = -self.rated
x = x / self.rated
if X_train3d.shape[0] == 0:
X_train3d = x.values
elif len(X_train3d.shape) == 2:
X_train3d = np.stack((X_train3d, x.values))
else:
X_train3d = np.vstack((X_train3d, x.values[np.newaxis, :, :]))
X_train3d = X_train3d[:, :, :, np.newaxis].transpose(1, 2, 0, 3)
combine_cnn = cnn_3d(comb_method_dir, self.rated)
combine_cnn.train(X_train3d, y1.values)