from dataprocessor import ProcessData from featureeng import Measures train = ProcessData.trainData() training_columns = list(train.columns) training_columns.remove('RUL') for column in training_columns: print(column, Measures.correlation(train[column].values, train['RUL'].values))
from dataprocessor import ProcessData import numpy as np import pandas as pd from sklearn.metrics import mean_squared_error # initialize server from featureeng.Math import moving_probability, probabilty_distribution h2o.init() # define response variable response_column = 'RUL' # load pre-processed data frames training_frame = ProcessData.trainData(standard_deviation=True, moving_k_closest_average=True) testing_frame = ProcessData.testData(standard_deviation=True, moving_k_closest_average=True) # create h2o frames train = h2o.H2OFrame(training_frame) test = h2o.H2OFrame(testing_frame) train.set_names(list(training_frame.columns)) test.set_names(list(testing_frame.columns)) # Feature selection training_columns = list(training_frame.columns) training_columns.remove(response_column) training_columns.remove("UnitNumber") training_columns.remove("Time")
# Recursive Feature Elimination import pandas as pd from sklearn.ensemble import RandomForestRegressor from sklearn.feature_selection import RFE from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error from sklearn_pandas import DataFrameMapper from dataprocessor import ProcessData, Filter training_frame = ProcessData.trainData() testing_frame = ProcessData.testData() # Remove anomalies in training frame based on percentile all_columns = list(training_frame.columns) rm_columns = ['UnitNumber', 'Time', 'Setting1', 'Setting2', 'Setting3', 'RUL'] filter_columns = [x for x in all_columns if x not in rm_columns] training_frame = Filter.filterDataPercentile(panda_frame=training_frame, columns=filter_columns, lower_percentile=0.01, upper_percentile=0.99, column_err_threshold=2) # Training data columns del training_frame['UnitNumber'] del training_frame['Time'] # Testing columns del testing_frame['UnitNumber'] del testing_frame['Time']
import math from dataprocessor import ProcessData from h2o.estimators import H2ODeepLearningEstimator from sklearn.metrics import mean_squared_error, mean_absolute_error # config _nmodels = 10 _smodels = 5 _lim = 1 _validation_ratio = 0.8 # initialize server h2o.init() # get processed data pTrain = ProcessData.trainData(moving_k_closest_average=True, standard_deviation=True, probability_distribution=True) pTest = ProcessData.testData(moving_k_closest_average=True, standard_deviation=True, probability_from_file=True) # convert to h2o frames hTrain = h2o.H2OFrame(pTrain) hTest = h2o.H2OFrame(pTest) hTrain.set_names(list(pTrain.columns)) hTest.set_names(list(pTest.columns)) # select column names response_column = 'RUL' training_columns = list(pTrain.columns) training_columns.remove(response_column) training_columns.remove("UnitNumber") training_columns.remove("Time")
import h2o import numpy as np from h2o.estimators import H2OAutoEncoderEstimator from h2o.estimators import H2ODeepLearningEstimator from h2o.estimators import H2ORandomForestEstimator from dataprocessor import ProcessData, Filter from featureeng import Measures from parser import DataFrameParser # Initialize server h2o.init() # AutoEncoder anomaly removal process p_train = ProcessData.trainData(bin_classification=True) p_test = ProcessData.testData(bin_classification=True) # Converting to h2o frane h_test = h2o.H2OFrame(p_test) h_test.set_names(list(p_test.columns)) h_train = h2o.H2OFrame(p_train) h_train.set_names(list(p_train.columns)) # Define autoencoder anomaly_model = H2OAutoEncoderEstimator(activation="Rectifier", hidden=[25, 12, 25], sparse=True, l1=1e-4, epochs=100) # Select relevant features
# Ridge Regression from sklearn import linear_model from sklearn.metrics import mean_squared_error, mean_absolute_error from sklearn_pandas import DataFrameMapper from sklearn.externals import joblib from dataprocessor import ProcessData import numpy as np import os # Define response variable response_column = "RUL" # Process data training_frame = ProcessData.trainData(moving_average=True, standard_deviation=True, probability_distribution=True) testing_frame = ProcessData.testData(moving_average=True, standard_deviation=True, probability_distribution=True) # Select training columns training_columns = list(training_frame.columns) training_columns.remove(response_column) # Remove RUL training_columns.remove("UnitNumber") # Remove UnitNumber training_columns.remove("Time") # Remove Time # Set mapper df_mapper = DataFrameMapper([(training_columns, None), (response_column, None)])
# Random Forest Regressor from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error, mean_absolute_error from sklearn_pandas import DataFrameMapper from sklearn.externals import joblib from dataprocessor import ProcessData import numpy as np import os # Define response variable response_column = "RUL" # Process data training_frame = ProcessData.trainData(moving_average=True, standard_deviation=True, moving_entropy=True) testing_frame = ProcessData.testData(moving_average=True, standard_deviation=True, moving_entropy=True) # Select training columns training_columns = list(training_frame.columns) training_columns.remove(response_column) # Remove RUL training_columns.remove("UnitNumber") # Remove UnitNumber training_columns.remove("Time") # Remove Time # Set mapper df_mapper = DataFrameMapper([(training_columns, None), (response_column, None)])
def function(): # AutoEncoder anomaly removal process p_train = ProcessData.trainData(moving_median_centered_average=True, standard_deviation=True, probability_distribution=True, bin_classification=True) p_test = ProcessData.testData(moving_median_centered_average=True, standard_deviation=True, probability_from_file=True, bin_classification=True) # Converting to h2o frane h_test = h2o.H2OFrame(p_test) h_test.set_names(list(p_test.columns)) h_train = h2o.H2OFrame(p_train) h_train.set_names(list(p_train.columns)) # Define autoencoder anomaly_model = H2OAutoEncoderEstimator(activation="Rectifier", hidden=[25, 12, 25], sparse=True, l1=1e-4, epochs=100) # Select relevant features anomaly_train_columns = list(p_train.columns) print(anomaly_train_columns) anomaly_train_columns.remove('RUL') anomaly_train_columns.remove('BIN') anomaly_train_columns.remove('UnitNumber') anomaly_train_columns.remove('Time') anomaly_train_columns.remove('Setting1') anomaly_train_columns.remove('Setting2') anomaly_train_columns.remove('Setting3') # Train model anomaly_model.train(x=anomaly_train_columns, training_frame=h_train) # Get reconstruction error reconstruction_error = anomaly_model.anomaly(test_data=h_train, per_feature=False) error_str = reconstruction_error.get_frame_data() err_list = list(map(float, error_str.split("\n")[1:-1])) err_list = np.array(err_list) # Threshold threshold = np.amax(err_list) * 0.97 print("Max Reconstruction Error :", reconstruction_error.max()) print("Threshold Reconstruction Error :", threshold) # Filter anomalies based on reconstruction error p_filter = Filter.filterDataAutoEncoder(panda_frame=p_train, reconstruction_error=err_list, threshold=threshold) # Drop features del p_filter['Setting3'] del p_filter['Sensor1'] del p_filter['Sensor5'] del p_filter['Sensor10'] del p_filter['Sensor16'] del p_filter['Sensor18'] del p_filter['Sensor19'] h_filter = h2o.H2OFrame(p_filter) h_filter.set_names(list(p_filter.columns)) h_test = h2o.H2OFrame(p_test) h_test.set_names(list(p_test.columns)) training_columns = list(p_filter.columns) training_columns.remove('UnitNumber') training_columns.remove('Time') training_columns.remove('RUL') training_columns.remove('BIN') h_filter['BIN'] = h_filter['BIN'].asfactor() h_test['BIN'] = h_test['BIN'].asfactor() model = H2ODeepLearningEstimator(variable_importances=True) model.train(x=training_columns, y='BIN', training_frame=h_filter, nfolds=10) predict = model.predict(test_data=h_test) predict = DataFrameParser.h2oToList(predict['predict']) actual = DataFrameParser.h2oToList(h_test['BIN']) Measures.confusion_matrix(actual, predict) print(predict) print(actual)
import h2o import numpy as np import math from dataprocessor import ProcessData from h2o.estimators import H2ODeepLearningEstimator from sklearn.metrics import mean_squared_error, mean_absolute_error # config _nmodels = 10 # initialize server h2o.init() # get processed data pTrain = ProcessData.trainData(moving_k_closest_average=True, standard_deviation=True) pTest = ProcessData.testData(moving_threshold_average=True, standard_deviation=True) # convert to h2o frames hTrain = h2o.H2OFrame(pTrain) hTest = h2o.H2OFrame(pTest) hTrain.set_names(list(pTrain.columns)) hTest.set_names(list(pTest.columns)) # select column names response_column = 'RUL' training_columns = list(pTrain.columns) training_columns.remove(response_column) training_columns.remove("UnitNumber")
# h2o testing import h2o from h2o.estimators import H2ODeepLearningEstimator from dataprocessor import ProcessData # initialize server from featureeng.Math import moving_probability, probabilty_distribution h2o.init() # define response variable response_column = 'RUL' # load pre-processed data frames training_frame = ProcessData.trainData(standard_deviation=True, moving_k_closest_average=True, moving_median_centered_average=True, moving_threshold_average=True) testing_frame = ProcessData.testData(standard_deviation=True, moving_k_closest_average=True, moving_median_centered_average=True, moving_threshold_average=True) # create h2o frames train = h2o.H2OFrame(training_frame) test = h2o.H2OFrame(testing_frame) train.set_names(list(training_frame.columns)) test.set_names(list(testing_frame.columns)) # Feature selection training_columns = list(training_frame.columns) training_columns.remove(response_column) training_columns.remove("UnitNumber") training_columns.remove("Time") # Building mode model = H2ODeepLearningEstimator(hidden=[200, 200], score_each_iteration=False, variable_importances=True)
from h2o.estimators import H2ODeepLearningEstimator from sklearn.metrics import mean_squared_error, mean_absolute_error from sklearn_pandas import DataFrameMapper from dataprocessor import ProcessData import math h2o.init() _nmodels = 10 _lim = 2 # define response variable response_column = 'RUL' # Pandas frame data_frame = ProcessData.trainData() test_frame = ProcessData.testData() # Create h2o frame h2o_data = h2o.H2OFrame(data_frame) h2o_data.set_names(list(data_frame.columns)) h2o_test = h2o.H2OFrame(test_frame) h2o_test.set_names(list(test_frame.columns)) # split frame data = h2o_data.split_frame(ratios=(0.9, 0.09)) # split data train_data = data[0] validate_data = data[1]