def get_classified_dataSets(X_ts,
labels_td,
model,
dropChannels,
COLUMN_ID='stopId'):
FP_X = X_ts
FN_X = X_ts
TP_X = X_ts
TN_X = X_ts
FP_y = labels_td
FN_y = labels_td
TP_y = labels_td
TN_y = labels_td
for stopId in X_ts[COLUMN_ID].unique():
currX = X_ts[X_ts[COLUMN_ID] == stopId]
currLabel = labels_td[labels_td[COLUMN_ID] == stopId]
X = pp.shape_Data_to_LSTM_format(currX, dropChannels)
y = pp.shape_Labels_to_LSTM_format(currLabel)
FP, FN, TP, TN = countTD_MaxLabel([(X, y)], model, silent=True)
if bool(FP):
FN_X = FN_X[FN_X[COLUMN_ID] != stopId]
TN_X = TN_X[TN_X[COLUMN_ID] != stopId]
TP_X = TP_X[TP_X[COLUMN_ID] != stopId]
FN_y = FN_y[FN_y[COLUMN_ID] != stopId]
TN_y = TN_y[TN_y[COLUMN_ID] != stopId]
TP_y = TP_y[TP_y[COLUMN_ID] != stopId]
elif bool(FN):
FP_X = FP_X[FP_X[COLUMN_ID] != stopId]
TN_X = TN_X[TN_X[COLUMN_ID] != stopId]
TP_X = TP_X[TP_X[COLUMN_ID] != stopId]
FP_y = FP_y[FP_y[COLUMN_ID] != stopId]
TN_y = TN_y[TN_y[COLUMN_ID] != stopId]
TP_y = TP_y[TP_y[COLUMN_ID] != stopId]
elif bool(TN):
FP_X = FP_X[FP_X[COLUMN_ID] != stopId]
FN_X = FN_X[FN_X[COLUMN_ID] != stopId]
TP_X = TP_X[TP_X[COLUMN_ID] != stopId]
FP_y = FP_y[FP_y[COLUMN_ID] != stopId]
FN_y = FN_y[FN_y[COLUMN_ID] != stopId]
TP_y = TP_y[TP_y[COLUMN_ID] != stopId]
elif bool(TP):
FP_X = FP_X[FP_X[COLUMN_ID] != stopId]
FN_X = FN_X[FN_X[COLUMN_ID] != stopId]
TN_X = TN_X[TN_X[COLUMN_ID] != stopId]
FP_y = FP_y[FP_y[COLUMN_ID] != stopId]
FN_y = FN_y[FN_y[COLUMN_ID] != stopId]
TN_y = TN_y[TN_y[COLUMN_ID] != stopId]
return (FP_X, FP_y), (FN_X, FN_y), (TP_X, TP_y), (TN_X, TN_y)
def get_overall_results(test_data, model, data_pd=False, dropChannels=None):
FP = 1
FN = 1
TP = 1
TN = 1
for currData in test_data:
if data_pd:
X_test = pp.shape_Data_to_LSTM_format(currData[0], dropChannels)
y = pp.shape_Labels_to_LSTM_format(currData[1])
else:
X_test = currData[0]
y = currData[1]
y_pred = model.predict_classes(X_test)
FP_loop, FN_loop, TP_loop, TN_loop = count_predictions(y, y_pred)
FP += FP_loop
FN += FN_loop
TP += TP_loop
TN += TN_loop
print('\nMCC: ' + str(get_MCC(FP, FN, TP, TN)))
print('\n 1_pr 0_pr')
print('\n 1_tr | ' + str(TP) + ' ' + str(FN))
print('\n 0_tr | ' + str(FP) + ' ' + str(TN))
return FP, FN, TP, TN
DataScaling = True
StopPatience = 30
for i in range(len(regularizers)):
RunName = RunNames[0]
filename = fileNames[0]
model = models[0]
rreg = regularizers[i]
breg = random.choice(regularizers)
kreg = random.choice(regularizers)
file = open(path + filename, 'rb')
Data = pickle.load(file)
dropChannels = ['time', 'stopId']
InputDataSet = pp.shape_Data_to_LSTM_format(Data[0][0], dropChannels)
input_shape = (None, InputDataSet.shape[2])
m = model_setup.modelDict[model](input_shape, rreg, breg, kreg)
histories = list()
testData = list()
batch_size = 10
epochs = 400
for currData in Data:
X_ts, labels = pp.balanceSlicedData(currData[0], currData[1], target=50, distributed_Output=True, COLUMN_ID='stopId')
TrainData, TestData = pp.splitDataPandasFormat(X_ts, labels, split=0.3)
X = pp.shape_Data_to_LSTM_format(TrainData[0], dropChannels, scale=DataScaling)
y = pp.shape_Labels_to_LSTM_format(TrainData[1])
callback = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=StopPatience, verbose=1,
mode='auto')
if X.shape[0] >= batch_size:
]
COLUMN_ID = 'stopId'
#COLUMN_ID = 'sliceId'
label_classified = pp.reduceLabel(
labels_td, indexStopID=True).to_frame().rename(columns={0: 'label'})
emptyDF = pd.DataFrame(np.zeros((len(label_classified), 4)),
index=label_classified.index,
columns=['P', 'N', 'set', 'eval'])
label_classified = pd.concat([label_classified, emptyDF], axis=1, sort=False)
for ModelName in ModelNameList:
m = load_model(ModelPath + ModelName + 'model.h5')
dropChannels = ['time', 'stopId']
for id in labels_td[COLUMN_ID].unique():
currX = pp.shape_Data_to_LSTM_format(X_ts.loc[X_ts[COLUMN_ID] == id])
y_pred = m.predict_classes(currX)
if bool(y_pred):
label_classified.loc[id, 'P'] = label_classified.loc[id, 'P'] + 1
else:
label_classified.loc[id, 'N'] = label_classified.loc[id, 'N'] + 1
thresholdP = 6
SavePath = 'Matlab/SingleWindow_AllSameId/'
SaveName = 'Classified_StopIdInfo'
ofile = open(SavePath + SaveName + '.csv', 'w')
writer = csv.writer(ofile, delimiter=",")
Infos = ['stopId', 'label', 'P', 'N', 'set', 'eval']
writer.writerow(Infos)
path = ''
Savepath = '/home/computations/ExperimentalData/ModelHistoryFiles'
DataScaling = True
StopPatience = 30
for i in range(len(RunNames)):
RunName = RunNames[i]
filename = fileNames[0]
model = models[i]
file = open(path + filename, 'rb')
Data = pickle.load(file)
dropChannels = ['time', 'stopId', 'trg1', 'n1', 'trot1', 'tlin1', 'tlin2', 'tamb1']
X, y = pp.balanceSlicedData(Data[0], Data[1], target=50, distributed_Output=False)
X = pp.shape_Data_to_LSTM_format(X, dropChannels, scale=DataScaling)
# y = pp.reduceLabel(y).values ## not needed if distributed Output in balance sliced Data == false
y = y.values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
class_weight = pp.getClassWeight_Dict(y)
epochs = 1
batch_size = 1
m = Sequential()
input_shape = (X.shape[1], X.shape[2])
m = model_setup.modelDict[model](input_shape)
callback = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=StopPatience, verbose=1, mode='auto')
history = m.fit(X_train, y_train, validation_split=0.2, epochs=epochs, batch_size=batch_size, verbose=2,
DataScaling = True
StopPatience = 30
for i in range(len(RunNames)):
RunName = RunNames[i]
model = models[0]
TrainFile = open(path+TrainNames[0], 'rb')
TestFile = open(path+TestNames[0], 'rb')
TrainData = pickle.load(TrainFile)
TestData = pickle.load(TestFile)
dropChannels = ['time', 'stopId', 'rh1', 'tempg', 'n1', 'tfld1', 'frc1', 'v1', 'trg1', 'trot1', 'dec1', 'tlin1', 'tlin2', 'tamb1']
dropChannels.append(additionalDropChannel[i])
X_train = pp.shape_Data_to_LSTM_format(TrainData[0], dropChannels, scale=DataScaling)
y_train = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TrainData[1]))
X_test = pp.shape_Data_to_LSTM_format(TestData[0], dropChannels, scale=DataScaling)
y_test = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TestData[1]))
epochs = 300
batch_size = 10
class_weight = {0: 1.,
1: 1.
}
m = Sequential()
input_shape = (X_train.shape[1], X_train.shape[2])
m = model_setup.modelDict[model](input_shape)
TestLabel = TestLabel.append(Data[1])
TestData = (TestData, TestLabel)
for name in TrainDataSets:
Data = pickle.load(open(path + name + '.p', 'rb'))
TrainData = TrainData.append(Data[0])
TrainLabel = TrainLabel.append(Data[1])
TrainData = (TrainData, TrainLabel)
DataScaling = True
StopPatience = 15
for i in range(len(RunNames)):
RunName = RunNames[i]
model = models[i]
X_train = pp.shape_Data_to_LSTM_format(TrainData[0],
dropChannels=dropChannels)
y_train = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TrainData[1]))
X_test = pp.shape_Data_to_LSTM_format(TestData[0],
dropChannels=dropChannels)
y_test = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TestData[1]))
epochs = 300
batch_size = 10
m = Sequential()
input_shape = (X_train.shape[1], X_train.shape[2])
m = model_setup.modelDict[model](input_shape)
callback = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
Testfilename = TestfileNames[0]
model = models[i]
file = open(path + Trainfilename, 'rb')
TrainData = pickle.load(file)
file = open(path + Testfilename, 'rb')
TestData = pickle.load(file)
TrainData = dropStopId_fromSet(TrainData[0],
TrainData[1],
dropStopId=dropStopId)
TestData = dropStopId_fromSet(TestData[0],
TestData[1],
dropStopId=dropStopId)
X_train = pp.shape_Data_to_LSTM_format(TrainData[0])
y_train = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TrainData[1]))
X_test = pp.shape_Data_to_LSTM_format(TestData[0])
y_test = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TestData[1]))
epochs = 300
batch_size = 10
m = Sequential()
input_shape = (X_train.shape[1], X_train.shape[2])
m = model_setup.modelDict[model](input_shape)
class_weight = {0: 2., 1: 1.}
callback = callbacks.EarlyStopping(monitor='val_loss',
import pickle
import numpy as np
import pandas as pd
from keras.models import load_model
from Libraries import data_preprocessing as pp
from Libraries import data_evaluation as d_Eval
ModelPath = '/media/computations/DATA/ExperimentalData/Runs/156417/'
ModelName = 'cross4L_16model'
m = load_model(ModelPath+ModelName+'.h5')
DataSetPath = '/media/computations/DATA/ExperimentalData/DataFiles/systemABCD/'
#TestDataSets = ['center8s_pad_B_TestDataPandas', 'center8s_pad_B_TrainDataPandas', 'center8s_pad_D_TestDataPandas', 'center8s_pad_D_TrainDataPandas', 'center8s_pad_C_TestDataPandas', 'center8s_pad_C_TrainDataPandas']
#TestDataSets = ['center8s_pad_TestDataPandas']
TestDataSets = ['center8s_pad_D_TestDataPandas']
TestData = pd.DataFrame()
TestLabel = pd.DataFrame()
dropChannels = ['time', 'stopId']
for name in TestDataSets:
Data = pickle.load(open(DataSetPath + name + '.p', 'rb'))
TestData = TestData.append(Data[0])
TestLabel = TestLabel.append(Data[1])
TestData = pp.shape_Data_to_LSTM_format(TestData, dropChannels=dropChannels)
TestLabel = pp.reduceNumpyTD(pp.shape_Labels_to_LSTM_format(TestLabel))
FP, FN, TP, TN = d_Eval.get_overall_results([(TestData, TestLabel)], m)
MCC = d_Eval.get_MCC(FP, FN, TP, TN)
#labels = pp.getTimeDistributedLabels(eec, X_ts)
loadPath = '/media/computations/DATA/ExperimentalData/Runs/135538/'
dataSetName = 'l10s_td_TestData'
Data = pickle.load(open(loadPath + dataSetName + '.p', 'rb'))
X_ts = Data[0]
labels = Data[1]
X_single = X_ts[X_ts['stopId'] == stopId]
labels_single = labels[labels['stopId'] == stopId]
#dropChannels = ['time', 'stopId', 'trg1', 'n1', 'trot1', 'tlin1', 'tlin2', 'tamb1']
dropChannels = ['time', 'stopId']
X = pp.shape_Data_to_LSTM_format(X_single, dropChannels)
y = pp.shape_Labels_to_LSTM_format(labels_single)
labels_td_pred = model.predict_classes(X)
FP, FN, TP, TN = d_eval.count_predictions(y, labels_td_pred)
print('\n 1_pr 0_pr')
print('\n 1_tr | ' + str(TP) + ' ' + str(FN))
print('\n 0_tr | ' + str(FP) + ' ' + str(TN))
d_eval.countTD_MaxLabel([(X, y)], model)
SavePath = 'Matlab/PredictedLabels1/'
SaveName = modelName + stopId + '_td_label'
ofile = open(SavePath + SaveName + '.csv', 'w')
import pickle
from keras.models import load_model
from Libraries import data_evaluation as d_eval
from Libraries import data_preprocessing as pp
runName = 'l6s_td'
runNr = '135538'
path = '/media/computations/DATA/ExperimentalData/Runs/' + runNr + '/'
model = load_model(path + runName + 'model.h5')
Data = pickle.load(open(path + runName + '_TestData.p', 'rb'))
X_test = Data[0]
y_test = Data[1]
## if test Data pandas
dropChannels = ['time', 'stopId']
X_test = pp.shape_Data_to_LSTM_format(Data[0], dropChannels, scale=True)
y_test = pp.shape_Labels_to_LSTM_format(Data[1])
FP, FN, TP, TN = d_eval.countTD_MaxLabel([(X_test, y_test)], model)
MCC = d_eval.get_MCC(FP, FN, TP, TN)
print('&' + str(MCC)[0:4] + '&' + str(TP) + '&' + str(TN) + '&' + str(FP) +
'&' + str(FN) + '\\' + '\\')
from sklearn.model_selection import train_test_split
from Libraries import model_setup
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import confusion_matrix
from keras.utils import plot_model
file = open('FindArchitecture/Dataset_1051/picklefiles/Data', 'rb')
Data = pickle.load(file)
x = Data[0]
m = Sequential()
dropChannels = [
'time', 'stopId', 'trg1', 'n1', 'trot1', 'tlin1', 'tlin2', 'tamb1'
]
InputDataSet = pp.shape_Data_to_LSTM_format(Data[0][0], dropChannels)
input_shape = (None, InputDataSet.shape[2])
m = model_setup.distributed_label(input_shape)
test_data = list()
epochs = 100
for currData in Data:
seed = 0
X = pp.shape_Data_to_LSTM_format(currData[0], dropChannels)
y = pp.shape_Labels_to_LSTM_format(currData[1])
#y = np.reshape(pp.reduceLabel(y).values, (X.shape[0], 1, 1))
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=seed)
batch_size = 5
if X_train.shape[0] >= 2:
weights = layer.get_weights()
inputs1 = Input(shape=(None, 8))
lstm1, state_h, state_c = LSTM(8, return_sequences=True,
return_state=True)(inputs1)
model = Model(inputs=inputs1, outputs=[lstm1, state_h, state_c])
model.set_weights(weights)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
Data = pickle.load(open(PATH + dataName, 'rb'))
dropChannels = [
'time', 'stopId', 'trg1', 'n1', 'trot1', 'tlin1', 'tlin2', 'tamb1'
]
X = pp.shape_Data_to_LSTM_format(Data[0], dropChannels)
Y = pp.shape_Labels_to_LSTM_format(Data[1])
fig = plt.figure()
ax1 = fig.add_subplot(1, 1, 1)
ax1.set_xlim([-1, 9])
ax1.set_ylim([-1, 4])
cm = plt.get_cmap('viridis')
cNorm = colors.Normalize(vmin=0, vmax=1)
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cm)
currX = X[0]
currY = Y[0]
plt.box(False)
Data = pickle.load(open(path+fileName+'.p', 'rb'))
X_ts = Data[0]
labels_td = Data[1]
COLUMN_ID = 'stopId'
#dropChannels = ['time', 'stopId', 'trg1', 'n1', 'trot1', 'tlin1', 'tlin2', 'tamb1']
dropChannels = ['time', 'stopId']
X_without, X_with, y_without, y_with = pp.getUpDownLabel_pandas(Data[0], Data[1], COLUMN_ID=COLUMN_ID)
y_with_np = pp.getUpDownLabels_np_v2(y_with, COLUMN_ID=COLUMN_ID)
y_without_np = pp.getUpDownLabels_np_v2(y_without, COLUMN_ID=COLUMN_ID)
X_with_np = pp.shape_Data_to_LSTM_format(X_with, dropChannels)
X_without_np = pp.shape_Data_to_LSTM_format(X_without, dropChannels)
input_shape = (None, X_with_np.shape[2])
m = model_setup.modelDict['m2l_16_UpDown'](input_shape)
epochs = 3
batch_size = 1
m.fit(X_with_np, y_with_np, validation_split=0.2, epochs=epochs, batch_size=batch_size, verbose=2)
def checkTimeSeriesUpDown(model, X_without_np):
label_pred = model.predict_classes(X_without_np)
up_count = 0
down_count = 0
false_squeal = 0