del training_frame[feature]
del testing_frame[feature]
selected_features = list(training_frame.columns)
selected_features.remove('UnitNumber')
selected_features.remove('Time')
selected_features.remove('RUL')
rm_columns = ['Setting1', 'Setting2', 'Setting3']
for col in rm_columns:
if col in training_frame and col in testing_frame:
selected_features.remove(col)
training_frame = ProcessData.trainDataToFrame(
training_frame=training_frame,
selected_column_names=selected_features,
moving_k_closest_average=True,
standard_deviation=True,
probability_distribution=True)
testing_frame = ProcessData.testDataToFrame(
testing_frame=testing_frame,
selected_column_names=selected_features,
moving_k_closest_average=True,
standard_deviation=True,
probability_from_file=True)
# Training data columns
del training_frame['UnitNumber']
del training_frame['Time']
# Testing columns
del testing_frame['UnitNumber']
series = pData[column]
anomaly = Test.iqr(series, threshold=5)
anomaly_series.extend(anomaly)
# Sort indexes
anomaly_series.sort()
anomaly_series = list(set(anomaly_series))
print anomaly_series
print len(anomaly_series)
# Remove anomalies
df = pData.drop(pData.index[anomaly_series])
# Feature engineering
data_frame = ProcessData.trainDataToFrame(df,
moving_k_closest_average=True,
standard_deviation=True)
testing_frame = ProcessData.testData(moving_k_closest_average=True,
standard_deviation=True)
# Create h2o frame
hData = h2o.H2OFrame(data_frame)
hData.set_names(list(data_frame.columns))
hTesting = h2o.H2OFrame(testing_frame)
hTesting.set_names(list(testing_frame.columns))
# Split data inti training and validation
hTrain, hValidate = hData.split_frame(ratios=[0.8])
h2o.export_file(hTrain, "hTrainMy.csv", force=True)
filtered_train = filtered_train.append(df1, ignore_index=True)
count += 1
Progress.printProgress(iteration=(i + 1),
total=hTrain.nrow,
decimals=1,
prefix="Progress",
suffix="Complete")
print filtered_train
print "Original Size :", hTrain.nrow
print "Filtered Size :", len(filtered_train)
print "Removed Rows :", (hTrain.nrow - len(filtered_train))
# Feature Engineering
pTrain = ProcessData.trainDataToFrame(filtered_train,
moving_k_closest_average=True,
standard_deviation=True,
probability_distribution=True)
pTest = ProcessData.testData(moving_k_closest_average=True,
standard_deviation=True,
probability_from_file=True)
# Convert pandas to h2o frame - for model training
hValidate = h2o.H2OFrame(pValidate)
hValidate.set_names(list(pValidate.columns))
hTrain = h2o.H2OFrame(pTrain)
hTrain.set_names(list(pTrain.columns))
hTest = h2o.H2OFrame(pTest)
hTest.set_names(list(pTest.columns))
filtered_train = pd.DataFrame()
count = 0
for i in trange(hTrain.nrow):
if abs(err_list[i] - q75) < 2 * iqr:
df1 = pTrain.iloc[i, :]
filtered_train = filtered_train.append(df1, ignore_index=True)
count += 1
print(filtered_train)
print("Original Size :", hTrain.nrow)
print("Filtered Size :", len(filtered_train))
print("Removed Rows :", (hTrain.nrow - len(filtered_train)))
# Feature Engineering
pTrain = ProcessData.trainDataToFrame(filtered_train,
moving_average=True,
standard_deviation=True)
pTest = ProcessData.testData(moving_average=True, standard_deviation=True)
# Convert pandas to h2o frame - for model training
hData = h2o.H2OFrame(pTrain)
hData.set_names(list(pTrain.columns))
hTrain, hValidate = hData.split_frame(ratios=[_validation_ratio_2])
hTest = h2o.H2OFrame(pTest)
hTest.set_names(list(pTest.columns))
# Save filtered frames
h2o.export_file(hTrain, "FilterTrain.csv", force=True)
h2o.export_file(hValidate, "FilterValidate.csv", force=True)
for column in all_columns:
if column not in sustain:
del train[column]
del test[column]
training_columns = sustain
training_columns.remove('UnitNumber')
training_columns.remove('RUL')
training_columns.remove('Time')
#filter_train = Process.filterData(panda_frame=train, columns=sustain, removal_method='iqr', threshold=4)
filter_train = train
feature_engineered_train = ProcessData.trainDataToFrame(
training_frame=filter_train,
moving_k_closest_average=True,
standard_deviation=True)
feature_engineered_test = ProcessData.trainDataToFrame(
training_frame=test,
moving_k_closest_average=True,
standard_deviation=True,
rul=True)
h_train = h2o.H2OFrame(feature_engineered_train)
h_train.set_names(list(feature_engineered_train.columns))
h_test = h2o.H2OFrame(feature_engineered_test)
h_test.set_names(list(feature_engineered_test.columns))
model = H2ODeepLearningEstimator(epochs=100,
series = pData[column]
anomaly = Test.iqr(series, threshold=3)
anomaly_series.extend(anomaly)
# Sort indexes
anomaly_series.sort()
anomaly_series = list(set(anomaly_series))
print anomaly_series
print len(anomaly_series)
# Remove anomalies
df = pData.drop(pData.index[anomaly_series])
# Feature engineering
data_frame = ProcessData.trainDataToFrame(df,
moving_k_closest_average=True,
standard_deviation=True,
probability_distribution=True)
testing_frame = ProcessData.testData(moving_k_closest_average=True,
standard_deviation=True,
probability_from_file=True)
# Create h2o frame
hData = h2o.H2OFrame(data_frame)
hData.set_names(list(data_frame.columns))
hTesting = h2o.H2OFrame(testing_frame)
hTesting.set_names(list(testing_frame.columns))
# Split data inti training and validation
hTrain, hValidate = hData.split_frame(ratios=[0.8])
del p_filter['Setting3']
del p_filter['Sensor1']
del p_filter['Sensor5']
del p_filter['Sensor10']
del p_filter['Sensor16']
del p_filter['Sensor18']
del p_filter['Sensor19']
# Feature engineering process
columns = [
'Sensor14', 'Sensor9', 'Sensor11', 'Sensor12', 'Sensor13', 'Sensor7',
'Sensor4', 'Sensor8', 'Sensor20', 'Sensor21', 'Sensor15', 'Sensor6',
'Sensor2', 'Sensor17', 'Sensor3'
]
p_featured_train = ProcessData.trainDataToFrame(training_frame=p_filter,
selected_column_names=columns,
probability_distribution=True)
p_featured_test = ProcessData.testDataToFrame(testing_frame=p_test,
selected_column_names=columns,
probability_from_file=True)
h_filter = h2o.H2OFrame(p_featured_train)
h_filter.set_names(list(p_featured_train.columns))
h_test = h2o.H2OFrame(p_featured_test)
h_test.set_names(list(p_featured_test.columns))
training_columns = list(p_featured_train.columns)
training_columns.remove('UnitNumber')
training_columns.remove('Time')
training_columns.remove('RUL')