def fit_model(self,
cvs,
method,
static_data,
cluster_dir,
optimize_method,
X_cnn=np.array([]),
X_lstm=np.array([]),
y=np.array([]),
rated=1):
# deap, optuna, skopt, grid_search
if optimize_method == 'deap':
from Fuzzy_clustering.ver_tf2.Sklearn_models_deap import sklearn_model
elif optimize_method == 'optuna':
from Fuzzy_clustering.ver_tf2.Sklearn_models_optuna import sklearn_model
elif optimize_method == 'skopt':
from Fuzzy_clustering.ver_tf2.Sklearn_models_skopt import sklearn_model
else:
from Fuzzy_clustering.ver_tf2.SKlearn_models import sklearn_model
# if (datetime.now().hour>=8 and datetime.now().hour<10):
# time.sleep(2*60*60)
if method == 'ML_RBF_ALL':
model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir)
model_rbf_ols = rbf_ols_module(cluster_dir,
rated,
static_data['sklearn']['njobs'],
GA=False)
model_rbf_ga = rbf_ols_module(cluster_dir,
rated,
static_data['sklearn']['njobs'],
GA=True)
if model_rbf_ols.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_ols')
self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf(cvs)
else:
self.models['RBF_OLS'] = model_rbf_ols.to_dict()
if model_rbf_ga.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_ga')
self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf(cvs)
else:
self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict()
if model_rbf.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_adam')
self.models['RBFNN'] = model_rbf.rbf_train(cvs)
else:
self.models['RBFNN'] = model_rbf.to_dict()
elif method == 'ML_RBF_ALL_CNN':
model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir)
model_rbf_ols = rbf_ols_module(cluster_dir,
rated,
static_data['sklearn']['njobs'],
GA=False)
model_rbf_ga = rbf_ols_module(cluster_dir,
rated,
static_data['sklearn']['njobs'],
GA=True)
if model_rbf_ols.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_ols')
self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf(cvs)
else:
self.models['RBF_OLS'] = model_rbf_ols.to_dict()
if model_rbf_ga.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_ga')
self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf(cvs)
else:
self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict()
if model_rbf.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_rbf_adam')
self.models['RBFNN'] = model_rbf.rbf_train(cvs)
else:
self.models['RBFNN'] = model_rbf.to_dict()
rbf_dir = [
model_rbf_ols.cluster_dir, model_rbf_ga.cluster_dir,
model_rbf.cluster_dir
]
model_cnn = cnn_model(static_data, rated, cluster_dir, rbf_dir)
if model_cnn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of model_cnn')
self.models['RBF-CNN'] = model_cnn.train_cnn(cvs)
else:
self.models['RBF-CNN'] = model_cnn.to_dict()
elif method == 'ML_NUSVM':
method = method.replace('ML_', '')
model_sklearn = sklearn_model(cluster_dir, rated, method,
static_data['sklearn']['njobs'])
if model_sklearn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of NUSVM')
self.models['NUSVM'] = model_sklearn.train(cvs)
else:
self.models['NUSVM'] = model_sklearn.to_dict()
elif method == 'ML_MLP':
method = method.replace('ML_', '')
model_sklearn = sklearn_model(cluster_dir, rated, method,
static_data['sklearn']['njobs'])
if model_sklearn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of MLP')
self.models['MLP'] = model_sklearn.train(cvs)
else:
self.models['MLP'] = model_sklearn.to_dict()
elif method == 'ML_SVM':
method = method.replace('ML_', '')
model_sklearn = sklearn_model(cluster_dir, rated, method,
static_data['sklearn']['njobs'])
if model_sklearn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of SVM')
self.models['SVM'] = model_sklearn.train(cvs)
else:
self.models['SVM'] = model_sklearn.to_dict()
elif method == 'ML_RF':
method = method.replace('ML_', '')
model_sklearn = sklearn_model(cluster_dir, rated, method,
static_data['sklearn']['njobs'])
if model_sklearn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of RF')
self.models['RF'] = model_sklearn.train(cvs)
else:
self.models['RF'] = model_sklearn.to_dict()
elif method == 'ML_XGB':
method = method.replace('ML_', '')
model_sklearn = sklearn_model(cluster_dir, rated, method,
static_data['sklearn']['njobs'])
if model_sklearn.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of XGB')
self.models['XGB'] = model_sklearn.train(cvs)
else:
self.models['XGB'] = model_sklearn.to_dict()
elif method == 'ML_CNN_3d':
cnn_model_3d = cnn_3d_model(static_data, rated, cluster_dir)
if cnn_model_3d.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of CNN_3d')
self.models['CNN_3d'] = cnn_model_3d.train_cnn(X_cnn, y)
else:
self.models['CNN_3d'] = cnn_model_3d.to_dict()
elif method == 'ML_LSTM_3d':
lstm_model_3d = lstm_3d_model(static_data, rated, cluster_dir)
if lstm_model_3d.istrained == False or static_data[
'train_online'] == True:
self.logger.info('Start of training of LSTM_3d')
self.models['LSTM_3d'] = lstm_model_3d.train_lstm(X_lstm, y)
else:
self.models['LSTM_3d'] = lstm_model_3d.to_dict()
self.save(self.cluster_dir)
def fit_model(self, cvs, method, static_data, cluster_dir, models, gpu, X_cnn=np.array([]), X_lstm=np.array([]), y=np.array([]), rated=1):
if method == 'ML_RBF_ALL':
model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir)
model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False)
model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True)
if model_rbf_ols.istrained==False or static_data['train_online']==True:
self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf_TL(cvs, models['RBF_OLS'])
else:
self.models['RBF_OLS'] = model_rbf_ols.to_dict()
if model_rbf_ga.istrained==False or static_data['train_online']==True:
self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf_TL(cvs, models['GA_RBF_OLS'])
else:
self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict()
if model_rbf.istrained==False or static_data['train_online']==True:
self.models['RBFNN'] = model_rbf.rbf_train_TL(cvs, models['RBFNN'], gpu)
else:
self.models['RBFNN'] = model_rbf.to_dict()
elif method == 'ML_RBF_ALL_CNN':
model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir)
model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False)
model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True)
if model_rbf_ols.istrained == False or static_data['train_online'] == True:
self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf_TL(cvs, models['RBF_OLS'])
else:
self.models['RBF_OLS'] = model_rbf_ols.to_dict()
if model_rbf_ga.istrained == False or static_data['train_online'] == True:
self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf_TL(cvs, models['GA_RBF_OLS'])
else:
self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict()
if model_rbf.istrained == False or static_data['train_online'] == True:
self.models['RBFNN'] = model_rbf.rbf_train_TL(cvs, models['RBFNN'], gpu)
else:
self.models['RBFNN'] = model_rbf.to_dict()
rbf_dir = [model_rbf_ols.cluster_dir, model_rbf_ga.cluster_dir, model_rbf.cluster_dir]
model_cnn = cnn_model(static_data, rated, cluster_dir, rbf_dir)
if model_cnn.istrained==False or static_data['train_online']==True:
self.models['RBF-CNN'] = model_cnn.train_cnn_TL(cvs, models['RBF-CNN'], gpu)
else:
self.models['RBF-CNN'] = model_cnn.to_dict()
elif method == 'ML_NUSVM':
method =method.replace('ML_','')
model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs'])
if model_sklearn.istrained==False or static_data['train_online']==True:
self.models['NUSVM'] = model_sklearn.train(cvs,models['NUSVM']['best_params'])
else:
self.models['NUSVM'] = model_sklearn.to_dict()
elif method == 'ML_MLP':
method = method.replace('ML_','')
model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs'])
if model_sklearn.istrained==False or static_data['train_online']==True:
self.models['MLP'] = model_sklearn.train(cvs, models['MLP']['best_params'])
else:
self.models['MLP'] = model_sklearn.to_dict()
elif method == 'ML_SVM':
method = method.replace('ML_','')
model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs'])
if model_sklearn.istrained==False or static_data['train_online']==True:
self.models['SVM'] = model_sklearn.train(cvs, models['SVM']['best_params'])
else:
self.models['SVM'] = model_sklearn.to_dict()
elif method == 'ML_RF':
method = method.replace('ML_','')
model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs'])
if model_sklearn.istrained==False or static_data['train_online']==True:
self.models['RF'] = model_sklearn.train(cvs, models['RF']['best_params'])
else:
self.models['RF'] = model_sklearn.to_dict()
elif method == 'ML_XGB':
method = method.replace('ML_','')
model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs'])
if model_sklearn.istrained==False or static_data['train_online']==True:
self.models['XGB'] = model_sklearn.train(cvs, models['XGB']['best_params'])
else:
self.models['XGB'] = model_sklearn.to_dict()
elif method == 'ML_CNN_3d':
cnn_model_3d = cnn_3d_model(static_data, rated, cluster_dir)
if cnn_model_3d.istrained == False or static_data['train_online'] == True:
self.models['CNN_3d'] = cnn_model_3d.train_cnn_TL(X_cnn, y, models['CNN_3d'], gpu)
else:
self.models['CNN_3d'] = cnn_model_3d.to_dict()
elif method == 'ML_LSTM_3d':
lstm_model_3d = lstm_3d_model(static_data, rated, cluster_dir)
if lstm_model_3d.istrained == False or static_data['train_online'] == True:
self.models['LSTM_3d'] = lstm_model_3d.train_lstm_TL(X_lstm, y, models['LSTM_3d'], gpu)
else:
self.models['LSTM_3d'] = lstm_model_3d.to_dict()
self.save(self.cluster_dir)
def __init__(self, static_data, rated, cluster_dir, rbf_dir):
self.static_data_all = static_data
self.static_data = static_data['CNN']
self.static_data_rbf = static_data['RBF']
self.rated = rated
self.cluster = os.path.basename(cluster_dir)
self.istrained = False
self.rbf_dir = rbf_dir
self.cluster_dir = cluster_dir
if isinstance(rbf_dir, list):
self.rbf_method = 'RBF_ALL'
self.cluster_cnn_dir = os.path.join(cluster_dir, 'RBF_ALL/CNN')
self.model_dir = os.path.join(self.cluster_cnn_dir, 'model')
self.rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir)
self.rbf.models=[]
for dir in rbf_dir:
rbf_method = os.path.basename(dir)
cluster_rbf_dir = os.path.join(dir, 'model')
if rbf_method == 'RBFNN':
rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir)
try:
rbf.load(cluster_rbf_dir)
except:
raise ImportError('Cannot load RBFNN models')
self.rbf.models.append(rbf.models[0])
elif rbf_method == 'RBF_OLS':
rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=False)
try:
rbf.load(cluster_rbf_dir)
except:
raise ImportError('Cannot load RBFNN models')
self.rbf.models.append(rbf.models[-1])
elif rbf_method == 'GA_RBF_OLS':
rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=True)
try:
rbf.load(cluster_rbf_dir)
except:
raise ImportError('Cannot load RBFNN models')
self.rbf.models.append(rbf.models[0])
else:
raise ValueError('Cannot recognize RBF method')
else:
self.rbf_method = os.path.basename(rbf_dir)
cluster_rbf_dir = os.path.join(rbf_dir, 'model')
self.cluster_cnn_dir = os.path.join(rbf_dir, 'CNN')
self.model_dir = os.path.join(self.cluster_cnn_dir, 'model')
if self.rbf_method == 'RBFNN':
self.rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir)
elif self.rbf_method == 'RBF_OLS':
self.rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=False)
elif self.rbf_method == 'GA_RBF_OLS':
self.rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=True)
else:
raise ValueError('Cannot recognize RBF method')
try:
self.rbf.load(cluster_rbf_dir)
except:
raise ImportError('Cannot load RBFNN models')
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
try:
self.load(self.model_dir)
except:
pass