from dataprocessor import ProcessData
from h2o.estimators import H2ODeepLearningEstimator
from sklearn.metrics import mean_squared_error, mean_absolute_error
# config
_nmodels = 10
_smodels = 5
_lim = 1
_validation_ratio = 0.8
# initialize server
h2o.init()
# get processed data
pTrain = ProcessData.trainData(moving_k_closest_average=True, standard_deviation=True, probability_distribution=True)
pTest = ProcessData.testData(moving_k_closest_average=True, standard_deviation=True, probability_from_file=True)
# convert to h2o frames
hTrain = h2o.H2OFrame(pTrain)
hTest = h2o.H2OFrame(pTest)
hTrain.set_names(list(pTrain.columns))
hTest.set_names(list(pTest.columns))
# select column names
response_column = 'RUL'
training_columns = list(pTrain.columns)
training_columns.remove(response_column)
training_columns.remove("UnitNumber")
training_columns.remove("Time")
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
# initialize server
from featureeng.Math import moving_probability, probabilty_distribution
h2o.init()
# define response variable
response_column = 'RUL'
# load pre-processed data frames
training_frame = ProcessData.trainData(standard_deviation=True,
moving_k_closest_average=True)
testing_frame = ProcessData.testData(standard_deviation=True,
moving_k_closest_average=True)
# create h2o frames
train = h2o.H2OFrame(training_frame)
test = h2o.H2OFrame(testing_frame)
train.set_names(list(training_frame.columns))
test.set_names(list(testing_frame.columns))
# Feature selection
training_columns = list(training_frame.columns)
training_columns.remove(response_column)
training_columns.remove("UnitNumber")
training_columns.remove("Time")
# Set mapper
df_mapper = DataFrameMapper([(training_columns, None),
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn_pandas import DataFrameMapper
from sklearn.externals import joblib
from dataprocessor import ProcessData
import numpy as np
import os
# Define response variable
response_column = "RUL"
# Process data
training_frame = ProcessData.trainData(moving_average=True,
standard_deviation=True,
probability_distribution=True)
testing_frame = ProcessData.testData(moving_average=True,
standard_deviation=True,
probability_distribution=True)
# Select training columns
training_columns = list(training_frame.columns)
training_columns.remove(response_column) # Remove RUL
training_columns.remove("UnitNumber") # Remove UnitNumber
training_columns.remove("Time") # Remove Time
# Set mapper
df_mapper = DataFrameMapper([(training_columns, None),
(response_column, None)])
# Train data - pandas to sklearn
data = df_mapper.fit_transform(training_frame)
# Recursive Feature Elimination
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import RFE
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn_pandas import DataFrameMapper
from dataprocessor import ProcessData, Filter
training_frame = ProcessData.trainData()
testing_frame = ProcessData.testData()
# Remove anomalies in training frame based on percentile
all_columns = list(training_frame.columns)
rm_columns = ['UnitNumber', 'Time', 'Setting1', 'Setting2', 'Setting3', 'RUL']
filter_columns = [x for x in all_columns if x not in rm_columns]
training_frame = Filter.filterDataPercentile(panda_frame=training_frame,
columns=filter_columns,
lower_percentile=0.01,
upper_percentile=0.99,
column_err_threshold=2)
# Training data columns
del training_frame['UnitNumber']
del training_frame['Time']
# Testing columns
del testing_frame['UnitNumber']
del testing_frame['Time']
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn_pandas import DataFrameMapper
from sklearn.externals import joblib
from dataprocessor import ProcessData
import numpy as np
import os
# Define response variable
response_column = "RUL"
# Process data
training_frame = ProcessData.trainData(moving_average=True,
standard_deviation=True,
moving_entropy=True)
testing_frame = ProcessData.testData(moving_average=True,
standard_deviation=True,
moving_entropy=True)
# Select training columns
training_columns = list(training_frame.columns)
training_columns.remove(response_column) # Remove RUL
training_columns.remove("UnitNumber") # Remove UnitNumber
training_columns.remove("Time") # Remove Time
# Set mapper
df_mapper = DataFrameMapper([(training_columns, None),
(response_column, None)])
# Train data - pandas to sklearn
data = df_mapper.fit_transform(training_frame)
import h2o
import numpy as np
from h2o.estimators import H2OAutoEncoderEstimator
from h2o.estimators import H2ODeepLearningEstimator
from h2o.estimators import H2ORandomForestEstimator
from dataprocessor import ProcessData, Filter
from featureeng import Measures
from parser import DataFrameParser
# Initialize server
h2o.init()
# AutoEncoder anomaly removal process
p_train = ProcessData.trainData(bin_classification=True)
p_test = ProcessData.testData(bin_classification=True)
# Converting to h2o frane
h_test = h2o.H2OFrame(p_test)
h_test.set_names(list(p_test.columns))
h_train = h2o.H2OFrame(p_train)
h_train.set_names(list(p_train.columns))
# Define autoencoder
anomaly_model = H2OAutoEncoderEstimator(activation="Rectifier",
hidden=[25, 12, 25],
sparse=True,
l1=1e-4,
epochs=100)
# Select relevant features
for i in trange(hTrain.nrow):
if abs(err_list[i] - q75) < 2 * iqr:
df1 = pTrain.iloc[i, :]
filtered_train = filtered_train.append(df1, ignore_index=True)
count += 1
print(filtered_train)
print("Original Size :", hTrain.nrow)
print("Filtered Size :", len(filtered_train))
print("Removed Rows :", (hTrain.nrow - len(filtered_train)))
# Feature Engineering
pTrain = ProcessData.trainDataToFrame(filtered_train,
moving_average=True,
standard_deviation=True)
pTest = ProcessData.testData(moving_average=True, standard_deviation=True)
# Convert pandas to h2o frame - for model training
hData = h2o.H2OFrame(pTrain)
hData.set_names(list(pTrain.columns))
hTrain, hValidate = hData.split_frame(ratios=[_validation_ratio_2])
hTest = h2o.H2OFrame(pTest)
hTest.set_names(list(pTest.columns))
# Save filtered frames
h2o.export_file(hTrain, "FilterTrain.csv", force=True)
h2o.export_file(hValidate, "FilterValidate.csv", force=True)
h2o.export_file(hTest, "FilterTest.csv", force=True)
def function():
# AutoEncoder anomaly removal process
p_train = ProcessData.trainData(moving_median_centered_average=True,
standard_deviation=True,
probability_distribution=True,
bin_classification=True)
p_test = ProcessData.testData(moving_median_centered_average=True,
standard_deviation=True,
probability_from_file=True,
bin_classification=True)
# Converting to h2o frane
h_test = h2o.H2OFrame(p_test)
h_test.set_names(list(p_test.columns))
h_train = h2o.H2OFrame(p_train)
h_train.set_names(list(p_train.columns))
# Define autoencoder
anomaly_model = H2OAutoEncoderEstimator(activation="Rectifier",
hidden=[25, 12, 25],
sparse=True,
l1=1e-4,
epochs=100)
# Select relevant features
anomaly_train_columns = list(p_train.columns)
print(anomaly_train_columns)
anomaly_train_columns.remove('RUL')
anomaly_train_columns.remove('BIN')
anomaly_train_columns.remove('UnitNumber')
anomaly_train_columns.remove('Time')
anomaly_train_columns.remove('Setting1')
anomaly_train_columns.remove('Setting2')
anomaly_train_columns.remove('Setting3')
# Train model
anomaly_model.train(x=anomaly_train_columns, training_frame=h_train)
# Get reconstruction error
reconstruction_error = anomaly_model.anomaly(test_data=h_train,
per_feature=False)
error_str = reconstruction_error.get_frame_data()
err_list = list(map(float, error_str.split("\n")[1:-1]))
err_list = np.array(err_list)
# Threshold
threshold = np.amax(err_list) * 0.97
print("Max Reconstruction Error :", reconstruction_error.max())
print("Threshold Reconstruction Error :", threshold)
# Filter anomalies based on reconstruction error
p_filter = Filter.filterDataAutoEncoder(panda_frame=p_train,
reconstruction_error=err_list,
threshold=threshold)
# Drop features
del p_filter['Setting3']
del p_filter['Sensor1']
del p_filter['Sensor5']
del p_filter['Sensor10']
del p_filter['Sensor16']
del p_filter['Sensor18']
del p_filter['Sensor19']
h_filter = h2o.H2OFrame(p_filter)
h_filter.set_names(list(p_filter.columns))
h_test = h2o.H2OFrame(p_test)
h_test.set_names(list(p_test.columns))
training_columns = list(p_filter.columns)
training_columns.remove('UnitNumber')
training_columns.remove('Time')
training_columns.remove('RUL')
training_columns.remove('BIN')
h_filter['BIN'] = h_filter['BIN'].asfactor()
h_test['BIN'] = h_test['BIN'].asfactor()
model = H2ODeepLearningEstimator(variable_importances=True)
model.train(x=training_columns,
y='BIN',
training_frame=h_filter,
nfolds=10)
predict = model.predict(test_data=h_test)
predict = DataFrameParser.h2oToList(predict['predict'])
actual = DataFrameParser.h2oToList(h_test['BIN'])
Measures.confusion_matrix(actual, predict)
print(predict)
print(actual)
from h2o.estimators import H2OAutoEncoderEstimator from h2o.estimators import H2ODeepLearningEstimator from sklearn.cross_validation import train_test_split from dataprocessor import ProcessData from featureeng import Progress _validation_ratio = 0.1 _reconstruction_error_rate = 0.9 # Define response column response_column = 'RUL' # initialize server h2o.init() # Load data frames pData = ProcessData.trainData() pTest = ProcessData.testData() # Split methods # method 1 train, validate = train_test_split(pData, test_size=_validation_ratio) print(len(train)) print(train) # method 2 validate = pData.sample(frac=_validation_ratio, random_state=200) train = pData.drop(validate.index) print(len(train)) print(train)
# h2o testing
import h2o
from h2o.estimators import H2ODeepLearningEstimator
from dataprocessor import ProcessData
# initialize server
from featureeng.Math import moving_probability, probabilty_distribution
h2o.init()
# define response variable
response_column = 'RUL'
# load pre-processed data frames
training_frame = ProcessData.trainData(standard_deviation=True, moving_k_closest_average=True, moving_median_centered_average=True, moving_threshold_average=True)
testing_frame = ProcessData.testData(standard_deviation=True, moving_k_closest_average=True, moving_median_centered_average=True, moving_threshold_average=True)
# create h2o frames
train = h2o.H2OFrame(training_frame)
test = h2o.H2OFrame(testing_frame)
train.set_names(list(training_frame.columns))
test.set_names(list(testing_frame.columns))
# Feature selection
training_columns = list(training_frame.columns)
training_columns.remove(response_column)
training_columns.remove("UnitNumber")
training_columns.remove("Time")
# Building mode
model = H2ODeepLearningEstimator(hidden=[200, 200], score_each_iteration=False, variable_importances=True)
from sklearn.metrics import mean_squared_error, mean_absolute_error from sklearn_pandas import DataFrameMapper from dataprocessor import ProcessData import math h2o.init() _nmodels = 10 _lim = 2 # define response variable response_column = 'RUL' # Pandas frame data_frame = ProcessData.trainData() test_frame = ProcessData.testData() # Create h2o frame h2o_data = h2o.H2OFrame(data_frame) h2o_data.set_names(list(data_frame.columns)) h2o_test = h2o.H2OFrame(test_frame) h2o_test.set_names(list(test_frame.columns)) # split frame data = h2o_data.split_frame(ratios=(0.9, 0.09)) # split data train_data = data[0] validate_data = data[1] test_data = h2o_test