def fit_model(self, cvs, method, static_data, cluster_dir, optimize_method, X_cnn=np.array([]), X_lstm=np.array([]), y=np.array([]), rated=1): # deap, optuna, skopt, grid_search if optimize_method == 'deap': from Fuzzy_clustering.ver_tf2.Sklearn_models_deap import sklearn_model elif optimize_method == 'optuna': from Fuzzy_clustering.ver_tf2.Sklearn_models_optuna import sklearn_model elif optimize_method == 'skopt': from Fuzzy_clustering.ver_tf2.Sklearn_models_skopt import sklearn_model else: from Fuzzy_clustering.ver_tf2.SKlearn_models import sklearn_model # if (datetime.now().hour>=8 and datetime.now().hour<10): # time.sleep(2*60*60) if method == 'ML_RBF_ALL': model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir) model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False) model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True) if model_rbf_ols.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_ols') self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf(cvs) else: self.models['RBF_OLS'] = model_rbf_ols.to_dict() if model_rbf_ga.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_ga') self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf(cvs) else: self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict() if model_rbf.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_adam') self.models['RBFNN'] = model_rbf.rbf_train(cvs) else: self.models['RBFNN'] = model_rbf.to_dict() elif method == 'ML_RBF_ALL_CNN': model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir) model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False) model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True) if model_rbf_ols.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_ols') self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf(cvs) else: self.models['RBF_OLS'] = model_rbf_ols.to_dict() if model_rbf_ga.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_ga') self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf(cvs) else: self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict() if model_rbf.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_rbf_adam') self.models['RBFNN'] = model_rbf.rbf_train(cvs) else: self.models['RBFNN'] = model_rbf.to_dict() rbf_dir = [ model_rbf_ols.cluster_dir, model_rbf_ga.cluster_dir, model_rbf.cluster_dir ] model_cnn = cnn_model(static_data, rated, cluster_dir, rbf_dir) if model_cnn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of model_cnn') self.models['RBF-CNN'] = model_cnn.train_cnn(cvs) else: self.models['RBF-CNN'] = model_cnn.to_dict() elif method == 'ML_NUSVM': method = method.replace('ML_', '') model_sklearn = sklearn_model(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of NUSVM') self.models['NUSVM'] = model_sklearn.train(cvs) else: self.models['NUSVM'] = model_sklearn.to_dict() elif method == 'ML_MLP': method = method.replace('ML_', '') model_sklearn = sklearn_model(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of MLP') self.models['MLP'] = model_sklearn.train(cvs) else: self.models['MLP'] = model_sklearn.to_dict() elif method == 'ML_SVM': method = method.replace('ML_', '') model_sklearn = sklearn_model(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of SVM') self.models['SVM'] = model_sklearn.train(cvs) else: self.models['SVM'] = model_sklearn.to_dict() elif method == 'ML_RF': method = method.replace('ML_', '') model_sklearn = sklearn_model(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of RF') self.models['RF'] = model_sklearn.train(cvs) else: self.models['RF'] = model_sklearn.to_dict() elif method == 'ML_XGB': method = method.replace('ML_', '') model_sklearn = sklearn_model(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of XGB') self.models['XGB'] = model_sklearn.train(cvs) else: self.models['XGB'] = model_sklearn.to_dict() elif method == 'ML_CNN_3d': cnn_model_3d = cnn_3d_model(static_data, rated, cluster_dir) if cnn_model_3d.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of CNN_3d') self.models['CNN_3d'] = cnn_model_3d.train_cnn(X_cnn, y) else: self.models['CNN_3d'] = cnn_model_3d.to_dict() elif method == 'ML_LSTM_3d': lstm_model_3d = lstm_3d_model(static_data, rated, cluster_dir) if lstm_model_3d.istrained == False or static_data[ 'train_online'] == True: self.logger.info('Start of training of LSTM_3d') self.models['LSTM_3d'] = lstm_model_3d.train_lstm(X_lstm, y) else: self.models['LSTM_3d'] = lstm_model_3d.to_dict() self.save(self.cluster_dir)
def fit_model(self, cvs, method, static_data, cluster_dir, models, gpu, X_cnn=np.array([]), X_lstm=np.array([]), y=np.array([]), rated=1): if method == 'ML_RBF_ALL': model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir) model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False) model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True) if model_rbf_ols.istrained==False or static_data['train_online']==True: self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf_TL(cvs, models['RBF_OLS']) else: self.models['RBF_OLS'] = model_rbf_ols.to_dict() if model_rbf_ga.istrained==False or static_data['train_online']==True: self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf_TL(cvs, models['GA_RBF_OLS']) else: self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict() if model_rbf.istrained==False or static_data['train_online']==True: self.models['RBFNN'] = model_rbf.rbf_train_TL(cvs, models['RBFNN'], gpu) else: self.models['RBFNN'] = model_rbf.to_dict() elif method == 'ML_RBF_ALL_CNN': model_rbf = rbf_model(static_data['RBF'], rated, cluster_dir) model_rbf_ols = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=False) model_rbf_ga = rbf_ols_module(cluster_dir, rated, static_data['sklearn']['njobs'], GA=True) if model_rbf_ols.istrained == False or static_data['train_online'] == True: self.models['RBF_OLS'] = model_rbf_ols.optimize_rbf_TL(cvs, models['RBF_OLS']) else: self.models['RBF_OLS'] = model_rbf_ols.to_dict() if model_rbf_ga.istrained == False or static_data['train_online'] == True: self.models['GA_RBF_OLS'] = model_rbf_ga.optimize_rbf_TL(cvs, models['GA_RBF_OLS']) else: self.models['GA_RBF_OLS'] = model_rbf_ga.to_dict() if model_rbf.istrained == False or static_data['train_online'] == True: self.models['RBFNN'] = model_rbf.rbf_train_TL(cvs, models['RBFNN'], gpu) else: self.models['RBFNN'] = model_rbf.to_dict() rbf_dir = [model_rbf_ols.cluster_dir, model_rbf_ga.cluster_dir, model_rbf.cluster_dir] model_cnn = cnn_model(static_data, rated, cluster_dir, rbf_dir) if model_cnn.istrained==False or static_data['train_online']==True: self.models['RBF-CNN'] = model_cnn.train_cnn_TL(cvs, models['RBF-CNN'], gpu) else: self.models['RBF-CNN'] = model_cnn.to_dict() elif method == 'ML_NUSVM': method =method.replace('ML_','') model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained==False or static_data['train_online']==True: self.models['NUSVM'] = model_sklearn.train(cvs,models['NUSVM']['best_params']) else: self.models['NUSVM'] = model_sklearn.to_dict() elif method == 'ML_MLP': method = method.replace('ML_','') model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained==False or static_data['train_online']==True: self.models['MLP'] = model_sklearn.train(cvs, models['MLP']['best_params']) else: self.models['MLP'] = model_sklearn.to_dict() elif method == 'ML_SVM': method = method.replace('ML_','') model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained==False or static_data['train_online']==True: self.models['SVM'] = model_sklearn.train(cvs, models['SVM']['best_params']) else: self.models['SVM'] = model_sklearn.to_dict() elif method == 'ML_RF': method = method.replace('ML_','') model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained==False or static_data['train_online']==True: self.models['RF'] = model_sklearn.train(cvs, models['RF']['best_params']) else: self.models['RF'] = model_sklearn.to_dict() elif method == 'ML_XGB': method = method.replace('ML_','') model_sklearn = sklearn_model_tl(cluster_dir, rated, method, static_data['sklearn']['njobs']) if model_sklearn.istrained==False or static_data['train_online']==True: self.models['XGB'] = model_sklearn.train(cvs, models['XGB']['best_params']) else: self.models['XGB'] = model_sklearn.to_dict() elif method == 'ML_CNN_3d': cnn_model_3d = cnn_3d_model(static_data, rated, cluster_dir) if cnn_model_3d.istrained == False or static_data['train_online'] == True: self.models['CNN_3d'] = cnn_model_3d.train_cnn_TL(X_cnn, y, models['CNN_3d'], gpu) else: self.models['CNN_3d'] = cnn_model_3d.to_dict() elif method == 'ML_LSTM_3d': lstm_model_3d = lstm_3d_model(static_data, rated, cluster_dir) if lstm_model_3d.istrained == False or static_data['train_online'] == True: self.models['LSTM_3d'] = lstm_model_3d.train_lstm_TL(X_lstm, y, models['LSTM_3d'], gpu) else: self.models['LSTM_3d'] = lstm_model_3d.to_dict() self.save(self.cluster_dir)
def __init__(self, static_data, rated, cluster_dir, rbf_dir): self.static_data_all = static_data self.static_data = static_data['CNN'] self.static_data_rbf = static_data['RBF'] self.rated = rated self.cluster = os.path.basename(cluster_dir) self.istrained = False self.rbf_dir = rbf_dir self.cluster_dir = cluster_dir if isinstance(rbf_dir, list): self.rbf_method = 'RBF_ALL' self.cluster_cnn_dir = os.path.join(cluster_dir, 'RBF_ALL/CNN') self.model_dir = os.path.join(self.cluster_cnn_dir, 'model') self.rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir) self.rbf.models=[] for dir in rbf_dir: rbf_method = os.path.basename(dir) cluster_rbf_dir = os.path.join(dir, 'model') if rbf_method == 'RBFNN': rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir) try: rbf.load(cluster_rbf_dir) except: raise ImportError('Cannot load RBFNN models') self.rbf.models.append(rbf.models[0]) elif rbf_method == 'RBF_OLS': rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=False) try: rbf.load(cluster_rbf_dir) except: raise ImportError('Cannot load RBFNN models') self.rbf.models.append(rbf.models[-1]) elif rbf_method == 'GA_RBF_OLS': rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=True) try: rbf.load(cluster_rbf_dir) except: raise ImportError('Cannot load RBFNN models') self.rbf.models.append(rbf.models[0]) else: raise ValueError('Cannot recognize RBF method') else: self.rbf_method = os.path.basename(rbf_dir) cluster_rbf_dir = os.path.join(rbf_dir, 'model') self.cluster_cnn_dir = os.path.join(rbf_dir, 'CNN') self.model_dir = os.path.join(self.cluster_cnn_dir, 'model') if self.rbf_method == 'RBFNN': self.rbf = rbf_model(self.static_data_rbf, self.rated, cluster_dir) elif self.rbf_method == 'RBF_OLS': self.rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=False) elif self.rbf_method == 'GA_RBF_OLS': self.rbf = rbf_ols_module(cluster_dir, rated, self.static_data_rbf['njobs'], GA=True) else: raise ValueError('Cannot recognize RBF method') try: self.rbf.load(cluster_rbf_dir) except: raise ImportError('Cannot load RBFNN models') if not os.path.exists(self.model_dir): os.makedirs(self.model_dir) try: self.load(self.model_dir) except: pass