def initializationProcess():
localprefix = '/home/'
serverprefix = '/home/labcompute/'
GVal.setPARA('prefix_PARA', serverprefix)
# GVal.setPARA('prefix_PARA', localprefix)
return GVal.getPARA('prefix_PARA')
def dataRegulationSKL(y_tra, X_tra, y_val, X_val, index_no):
kick_off_no1 = GVal.getPARA('kick_off_no1_PARA')
weights_on = GVal.getPARA('weights_on_PARA')
weight_list = GVal.getPARA('weight_list_PARA')
# print(max(weights), min(weights))
if weights_on:
weights = copy.deepcopy(y_tra)
for key in weight_list.keys():
weights[np.nonzero(weights == key)[0]] = weight_list[key]
else:
weights = np.ones(y_tra.shape)
kick_off_no1_switcher = {
0: ko1processor0,
1: ko1processor1,
2: ko1processor2,
3: ko1processor3,
4: ko1processor4
}
y_tra, X_tra, y_val, X_val, weights = kick_off_no1_switcher[GVal.getPARA(
'kick_off_no1_PARA')](y_tra, X_tra, y_val, X_val, weights, index_no)
# Change the staged level 0,1,2,3 into a binary situation 0 and 1(1,2,3)
y_tra[np.nonzero(y_tra > 0)[0]] = 1
y_val[np.nonzero(y_val > 0)[0]] = 1
return y_tra, X_tra, y_val, X_val, weights
def processCodeDecoder(process_code, decoder_update_flag):
process_code_str = str(process_code)
classifier_num_temp = int(process_code_str[-3:])
recording_index_list_selectserial = int(process_code_str[-5:-3]) - 1
recording_index_list = GVal.getPARA('recording_index_list_PARA')
loopPARA_bit = GVal.getPARA('loopPARA_bit_PARA')
loopPARA_decibit = GVal.getPARA('loopPARA_decibit_PARA')
if not process_code_str[-5 - loopPARA_bit:-5]:
loopPARA_value_index = 0
else:
loopPARA_value_index = float(process_code_str[-5 - loopPARA_bit:-5])
GVal.setPARA(GVal.getPARA('loopPARA_name'),
loopPARA_value_index / (10**loopPARA_decibit))
select_index_temp = recording_index_list[recording_index_list_selectserial]
if decoder_update_flag:
GVal.setPARA('classifier_num_PARA', classifier_num_temp)
GVal.setPARA('select_index_PARA', select_index_temp)
print(
'### Decoder Success! New parameters: [select_index] and [classifier_num] are updated into the workspace! '
)
else:
print(
'### Decoder Success! Output parameters: [select_index] and [classifier_num] '
)
print(
'### Caution! The parameters are not updated into the workspace! ')
return select_index_temp, classifier_num_temp
def caseRawsplit(X, Y, para):
test_portion = para
if test_portion > 1 or test_portion < 0:
print(
'Warning! The input test_portion is not in range[0,1], set the default test_portion as 0.25'
)
testportion = 0.25
N = len(GVal.getPARA('random_seed_PARA'))
for i in range(N):
X_train_temp, X_test_temp, y_train_temp, y_test_temp = skmdls.train_test_split(
X,
Y,
test_size=test_portion,
random_state=int(GVal.getPARA('random_seed_PARA')[i]))
# print(X_train_temp.shape)
# print(X_test_temp.shape)
# print(y_train_temp.shape)
# print(y_train_temp.shape)
# print('$@%' * 40)
if i == 0:
X_train = [X_train_temp]
X_test = [X_test_temp]
y_train = [y_train_temp]
y_test = [y_test_temp]
else:
X_train += [X_train_temp]
X_test += [X_test_temp]
y_train += [y_train_temp]
y_test += [y_test_temp]
return X_train, X_test, y_train, y_test, N
def labelProcessor(X0, Y0, X0V, Y0V):
# labelProcessor is working on processing the noconfident frame problem
# If a confidential information is attached with the label. This function can be applied to process.
# There several process methods below. Please add any customized processor in any existed or new switcher and write the funtion above in this file
path = GVal.getPARA('path_PARA')
process_code = GVal.getPARA('process_code_PARA')
# h = plt.figure(num=1, figsize=(17, 9.3))
# plt.subplot(211)
# plt.plot(Y0[:, 2], label='Noise Label ')
# plt.plot(Y0[:, 1], '*', label='Confident')
# plt.plot(Y0[:, 0], label='Class Label')
# plt.legend()
FLAG = GVal.getPARA('FLAG_PARA')
if FLAG['noise_frame_dilation'] == 1:
YnFD = noiseFrameDilation(Y0[:, 2])
# Rewrite back into the label variable.
for iY in range(len(Y0[:, 2])):
Y0[iY, 2] = YnFD[iY]
noconfident_frame_process_switcher = {
0: [processor0, 'Do nothing'],
1: [processor1, 'Change label 8 to 7'],
2: [processor2, 'Change label 7 to 8'],
3: [processor3, 'Delete label 7'],
4: [processor4, 'Delete label 8'],
5: [processor5, 'Deleta label 7&8']
}
noconfident_frame_process = GVal.getPARA('noconfident_frame_process_PARA')
print('######## [ Abnormal Frame Processing ] ######## ')
print('### Current Data Size: [X]--' + str(X0.shape) + ' [Y]--' + str(Y0.shape))
X1, Y1 = noconfident_frame_process_switcher[noconfident_frame_process][0](X0, Y0, noconfident_frame_process_switcher[noconfident_frame_process][1])
noise_frame_process_switcher = {
0: [processor0, 'Do nothing'],
1: [processor6, 'Delete noise frame'],
2: [processor7, 'Delete noise frame and noblink frame'],
}
noise_frame_process = GVal.getPARA('noise_frame_process_PARA')
print('### [ Training Set ] Current Data Size: [X]--' + str(X1.shape) + ' [Y]--' + str(Y1.shape))
X_out, Y_out = noise_frame_process_switcher[noise_frame_process][0](X1, Y1, noise_frame_process_switcher[noise_frame_process][1])
print('### [ Training Set ] Current Data Size: [X]--' + str(X_out.shape) + ' [Y]--' + str(Y_out.shape))
print('### [ Validation Set ] Current Data Size: [X]--' + str(X0V.shape) + ' [Y]--' + str(Y0V.shape))
XV_out, YV_out = noise_frame_process_switcher[noise_frame_process][0](X0V, Y0V, noise_frame_process_switcher[noise_frame_process][1])
print('### [ Validation Set ] Current Data Size: [X]--' + str(XV_out.shape) + ' [Y]--' + str(YV_out.shape))
# plt.subplot(212)
# plt.plot(Y_out[:, 2], label='Noise Label ')
# plt.plot(Y_out[:, 1], '*', label='Confident')
# plt.plot(Y_out[:, 0], label='Class Label')
# plt.legend()
# plt.show()
# plt.savefig((path['work_path'] + 'AbnormalFrameProcess_' + str(process_code) + '.png'))
# print('### The label processing result comprison figure is saved in workpath!')
return X_out, Y_out, XV_out, YV_out
def featurePlotting(label_all, online_fea_all, processCode):
nega_index_raw = np.nonzero(label_all[0] == 0)[0]
posi1_index_raw = np.nonzero(label_all[0] == 1)[0]
posi2_index_raw = np.nonzero(label_all[0] == 2)[0]
posi3_index_raw = np.nonzero(label_all[0] == 3)[0]
# [nega, posi1, posi2, posi3]
colorlist1 = ['#000000', '#000000', '#0d75f8', '#e50000']
nega_index = nega_index_raw
posi1_index = posi1_index_raw
posi2_index = posi2_index_raw
posi3_index = posi3_index_raw
screen_fea_list = GVal.getPARA('screen_fea_list_PARA')
online_fea_name = GVal.getPARA('online_fea_name_PARA')
print(len(nega_index), len(posi1_index), len(posi2_index),
len(posi3_index))
pic_num = 1
for fea1_n in range(len(screen_fea_list)):
fea1_num = screen_fea_list[fea1_n]
for fea2_n in range(fea1_n + 1, len(screen_fea_list)):
fea2_num = screen_fea_list[fea2_n]
figcode = int(1e13 * pic_num + processCode)
h = plt.figure(num=figcode, figsize=(17, 9.3))
plt.subplot(131)
plt.plot(online_fea_all[fea1_n][nega_index],
online_fea_all[fea2_n][nega_index],
'o',
color=colorlist1[0],
label='No Fatigue')
plt.plot(online_fea_all[fea1_n][posi1_index],
online_fea_all[fea2_n][posi1_index],
'.',
color=colorlist1[1],
label='Fatigue LV7')
plt.plot(online_fea_all[fea1_n][posi2_index],
online_fea_all[fea2_n][posi2_index],
'o',
color=colorlist1[2],
label='Fatigue LV8')
plt.plot(online_fea_all[fea1_n][posi3_index],
online_fea_all[fea2_n][posi3_index],
'o',
color=colorlist1[3],
label='Fatigure LV9')
plt.legend(loc='best', prop=zhfont)
plt.xlabel(online_fea_name[fea1_num], FontProperties=zhfont)
plt.ylabel(online_fea_name[fea2_num], FontProperties=zhfont)
plt.title('Figure #' + str(figcode))
plt.show()
plt.close(h)
# plt.savefig((work_path + '/pic/Figure' + str(pic_num) + '.png'))
# print('Picture' + str(pic_num) + 'Saved!')
# plt.close(h)
pic_num += 1
return 0
def feaSelection(X):
feaSelectionStrategy = GVal.getPARA('feaSelectionStrategy')
nComponent = GVal.getPARA('nComponent_PARA')
feaSelectionStrategies = {1: tSNE, 2: normalPCA}
X_fss = feaSelectionStrategies[feaSelectionStrategy](X, nComponent)
GVal.setPARA('feature_index_PARA',
np.arange(0, GVal.getPARA('nComponent_PARA')))
return X_fss
def labelProcessor(X0, Y0):
path = GVal.getPARA('path_PARA')
process_code = GVal.getPARA('process_code_PARA')
h = plt.figure(num=1, figsize=(17, 9.3))
plt.subplot(211)
plt.plot(Y0[:, 2], label='Noise Label ')
plt.plot(Y0[:, 1], '*', label='Confident')
plt.plot(Y0[:, 0], label='Class Label')
plt.legend()
noconfident_frame_process_switcher = {
0: [processor0, 'Do nothing'],
1: [processor1, 'Change label 0 to 7'],
2: [processor2, 'Change label 7 to 0'],
3: [processor3, 'Delete label 7'],
4: [processor4, 'Delete label 0'],
5: [processor5, 'Deleta label 7&0']
}
noconfident_frame_process = GVal.getPARA('noconfident_frame_process_PARA')
print('######## [ Abnormal Frame Processing ] ######## ')
print('### Current Data Size: [X]--' + str(X0.shape) + ' [Y]--' +
str(Y0.shape))
X1, Y1 = noconfident_frame_process_switcher[noconfident_frame_process][0](
X0, Y0,
noconfident_frame_process_switcher[noconfident_frame_process][1])
noise_frame_process_switcher = {
0: [processor0, 'Do nothing'],
1: [processor6, 'Delete noise frame'],
2: [processor7, 'Delete noise frame and noblink frame'],
}
noise_frame_process = GVal.getPARA('noise_frame_process_PARA')
print('### Current Data Size: [X]--' + str(X1.shape) + ' [Y]--' +
str(Y1.shape))
X_out, Y_out = noise_frame_process_switcher[noise_frame_process][0](
X1, Y1, noise_frame_process_switcher[noise_frame_process][1])
print('### Current Data Size: [X]--' + str(X_out.shape) + ' [Y]--' +
str(Y_out.shape))
plt.subplot(212)
plt.plot(Y_out[:, 2], label='Noise Label ')
plt.plot(Y_out[:, 1], '*', label='Confident')
plt.plot(Y_out[:, 0], label='Class Label')
plt.legend()
plt.show()
plt.savefig((path['work_path'] + 'AbnormalFrameProcess_' +
str(process_code) + '.png'))
print(
'### The label processing result comprison figure is saved in workpath!'
)
return X_out, Y_out
def processLearning(mdl, X_tra, y_tra, X_val, y_val):
print('######## [Predicting ... ] ########')
# Z is the classification result with the current model (mdl,no matter what kind of classifier)
Z = mdl.predict(X_val)
score = mdl.score(X_tra, y_tra)
FRAP = calculateFRAP(Z, y_val, 1)
Z_tra = mdl.predict(X_tra)
GVal.setPARA('Z_tra_res_PARA', Z_tra)
GVal.setPARA('Z_res_PARA', Z)
return mdl, score, FRAP
def dataBalance(X, Y):
data_balance_process = GVal.getPARA('data_balance_process_PARA')
data_balance_para = GVal.getPARA('data_balance_para_PARA')
data_balance_frame = {
1: downSamplingNega,
}
XX, YY = data_balance_frame[data_balance_process](X, Y, data_balance_para)
return XX, YY
def processCodeEncoder():
recording_index_list = GVal.getPARA('recording_index_list_PARA')
classifier_list = GVal.getPARA('classifier_list_PARA')
loopPARA_valuelist = GVal.getPARA('loopPARA_valuelist_PARA')
loopPARA_bit = GVal.getPARA('loopPARA_bit_PARA')
process_code_pack = []
for recording_index_list_lplb in np.arange(len(recording_index_list)):
for classifier_list_lplb in np.arange(len(classifier_list)):
for loopPARA_valuelist_lplb in np.arange(len(loopPARA_valuelist)):
code_temp = int(1e0 * classifier_list[classifier_list_lplb] +
1e3 * (recording_index_list_lplb + 1) + 1e5 *
loopPARA_valuelist[loopPARA_valuelist_lplb])
process_code_pack.append(code_temp)
return process_code_pack
def cFRAPPlotting(res):
L_clf = len(GVal.getPARA('classifier_list_PARA'))
if L_clf < 2:
print(
'### Warning! No enough classifiers for cFRAP Plotting, skip it.')
return 0
ft_size = 18
path = GVal.getPARA('path_PARA')
beta = GVal.getPARA('beta_PARA')
y = np.zeros((5, L_clf))
xlbl = cell2dmatlab_jsp([L_clf], 1, [])
for i in range(L_clf):
xlbl[i] = res[i][1]
for j in range(3):
y[j][i] = res[i][3][j]
for jj in range(5, 7):
y[jj - 2][i] = res[i][3][jj]
ylbl = [
'[P] Precision', '[A] Accuracy', '[R] Recall', '[F1] score',
['[F' + str(beta) + '] score']
]
colorlist = ['#0203e2', '#6ecb3c', '#fd3c06', '#000000', '#000000']
h = plt.figure(num=1, figsize=(17, 9.3))
ax = plt.gca()
port = 0.1
ytick = np.arange(0, 1, 0.1)
x = np.arange(1, L_clf + 1, 1)
for j in range(5):
delt = port * j + 0.01 * j
plt.bar(x - 0.3 + delt,
y[j],
width=port,
facecolor=colorlist[j],
label=ylbl[j])
plt.legend(loc=2, fontsize=ft_size)
ax.set_xticks(x)
ax.set_yticks(ytick)
ax.set_xticklabels(xlbl, fontproperties=zhfont, fontsize=ft_size)
ax.set_yticklabels(ytick, fontsize=ft_size)
plt.ylabel('scores', fontsize=ft_size)
plt.ylim(0, 1.05)
plt.grid()
plt.show()
plt.savefig((path['fig_path'] + 'cFRAP.png'))
print('Picture for ' + str(L_clf) + ' Various Classifers Saved!')
def kerasTrainer(classifier_num, X_train_raw, Y_train_raw, X_valid_raw,
Y_valid_raw, path):
feature_index = GVal.getPARA('feature_index_PARA')
X, y, X_valid, y_valid, index_no = dataSetPreparation(
feature_index, X_train_raw, Y_train_raw, X_valid_raw, Y_valid_raw)
classifier_list = {
11: [sequentialNN, 'Sequential Frame NN'],
12: [sequentialNN, 'Sequential Frame NN']
}
clf_cache = {
11: cell2dmatlab_jsp([1], 1, []),
12: cell2dmatlab_jsp([1], 1, []),
}
print('### With model: [' + classifier_list[classifier_num][1] + ']')
print('######## [Predicting ... ] ########')
# Loading model to do the classification
clf, score, FRAP = classifier_list[int(str(classifier_num)[0:2])][0](
X, y, X_valid, y_valid, index_no, classifier_num)
clf_cache[classifier_num] = clf
# return clf,score,FRAP
return classifier_list[classifier_num][1], score, FRAP
def plainCodePool(nlevel, dims, temp, h, y):
LP_nc = GVal.getPARA('loopPARA_namecache_PARA')
dV = cell2dmatlab_jsp([len(LP_nc)], 1, [])
for n in range(len(LP_nc)):
if type(LP_nc[n][0]) == int:
dV[n] = dVM[LP_nc[n][0]][2]
else:
dV[n] = GVal.getDVPARA(LP_nc[n][0])
for i in range(nlevel):
for j in range(dims[i]):
tempdV = copy.deepcopy(dV)
tempdV[i] = h[i][j]
y = np.vstack((y, tempdV))
return y
def randomForest(X_tra, y_tra, X_val, y_val, index_no, classifier_num):
y_tra, X_tra, y_val, X_val, weights = dataRegulationSKL(
y_tra, X_tra, y_val, X_val, index_no)
# http://blog.csdn.net/xuxiatian/article/details/54410086
clf = skemb.RandomForestClassifier(n_estimators=dVM[3200][2],
criterion=dVM[3201][2],
max_features=dVM[3202][2],
max_depth=dVM[3203][2],
min_samples_split=dVM[3204][2],
min_samples_leaf=dVM[3205][2],
min_weight_fraction_leaf=dVM[3206][2],
random_state=dVM[3213][2])
# GVal.show('dVM_PARA')
clf.fit(X_tra, y_tra, sample_weight=weights)
# print(clf.get_params())
# print(clf)
path = GVal.getPARA('path_PARA')
i_tree = 0
for tree_in_forest in clf.estimators_:
with open(path['fig_path'] + '/RF/tree_' + str(i_tree) + '.dot',
'w') as my_file:
my_file = sktree.export_graphviz(tree_in_forest,
out_file=my_file,
class_names=['0', '1'])
i_tree = i_tree + 1
return processLearning(clf, X_tra, y_tra, X_val, y_val)
def processCodeDecoder(process_code):
print('### Input process_code is: [' + str(process_code) + ']')
process_code_str = str(process_code)
classifier_num_temp = int(process_code_str[-3:])
recording_index_list_selectserial = int(process_code_str[-5:-3]) - 1
recording_index_list = GVal.getPARA('recording_index_list_PARA')
loopPARA_amount = GVal.getPARA('loopPARA_amount_PARA')
if loopPARA_amount > 0:
loopPARA_codepool = GVal.getPARA('loopPARA_codepool_PARA')
loopPARA_namecache = GVal.getPARA('loopPARA_namecache_PARA')
loopPARA_serialnum = int(process_code_str[-11:-7])
for loopPARA_num in range(loopPARA_amount):
# Set the value in current loop according to the process_code
GVal.setPARA(loopPARA_namecache[loopPARA_num][0],
loopPARA_codepool[loopPARA_serialnum][loopPARA_num])
if type(loopPARA_namecache[loopPARA_num][0]) == int:
dVM[loopPARA_namecache[loopPARA_num][0]][2] = str2num(
loopPARA_codepool[loopPARA_serialnum][loopPARA_num])
print(
'### Looping Parameter (' + str(loopPARA_num + 1) + '/' +
str(loopPARA_amount) + '): [' +
str(loopPARA_namecache[loopPARA_num][0]) + ' | ' +
dVM[loopPARA_namecache[loopPARA_num][0]][0] +
'], with value set: [' +
str(loopPARA_codepool[loopPARA_serialnum][loopPARA_num]) +
']')
else:
print(
'### Looping Parameter (' + str(loopPARA_num + 1) + '/' +
str(loopPARA_amount) + '): [' +
str(loopPARA_namecache[loopPARA_num][0]) +
'], with value set: [' +
str(loopPARA_codepool[loopPARA_serialnum][loopPARA_num]) +
']')
select_index_temp = recording_index_list[recording_index_list_selectserial]
GVal.setPARA('classifier_num_PARA', classifier_num_temp)
GVal.setPARA('select_index_PARA', select_index_temp)
print('######## [ Decoder Success! ] ########')
return select_index_temp, classifier_num_temp
def featurePreparation(select_index, online_fea_selectindex, noise_label_index,
active_index, data_file):
# Concatenate all the subjects together in one huge matrix.
# Comprehensive feature preparation.
# Bluring the concept of subjects. Combine all records together.
# [TODO] Add the Developing Features
dev_fea_signal = []
GVal.initPARA('online_fea_raw_cache', {'name': 'data'})
online_fea_signal = cell2dmatlab_jsp(
[len(select_index), len(online_fea_selectindex)], 2, [])
noise_label_signal = cell2dmatlab_jsp([len(select_index), 2], 2, [])
lplb_count1 = 0
for i in select_index:
if i not in active_index:
lplb_count1 += 1
continue
recordname = data_file[i]
# Loading of the online features (Developed)
online_fea_file = glob.glob(path['online_fea_path'] + recordname +
'*.txt')
if recordname in GVal.getPARA('online_fea_raw_cache').keys():
online_fea_raw = GVal.getPARA('online_fea_raw_cache')[recordname]
else:
online_fea_raw = np.loadtxt(online_fea_file[0])
GVal.getPARA('online_fea_raw_cache')[recordname] = online_fea_raw
# Noise label information. (Contain noise and noblink label information)
j_lplb_count = 0
for j in noise_label_index:
noise_label_signal[lplb_count1][j_lplb_count] = online_fea_raw[:,
j]
j_lplb_count += 1
# Online Developed Features
j_lplb_count = 0
for j in online_fea_selectindex:
online_fea_signal[lplb_count1][j_lplb_count] = online_fea_raw[:, j]
j_lplb_count += 1
# exit()
lplb_count1 += 1
# Save for FINAL EVALUATION
# GVal.setPARA('feas_PARA',online_fea_raw[])
GVal.setPARA('online_fea_signal_PARA', online_fea_signal)
GVal.setPARA('noise_label_signal_PARA', noise_label_signal)
return online_fea_signal, dev_fea_signal, noise_label_signal
def labelReading_NFDA(select_index, data_file, path):
# Initialize several matrix
label_signal = cell2dmatlab_jsp([len(select_index), 2], 2, [])
subject_sample_length = np.zeros([len(select_index), 1])
active_index = []
GVal.initPARA('label_raw_cache', {'name': 'data'})
# Iplb LooPLaBel ,for the for loop
lplb_count1 = 0
for i in select_index:
recordname = data_file[i]
print(recordname)
# Loading of labels.
label_file = glob.glob(path['label_path'] + recordname + '*.txt')
if not label_file:
lplb_count1 += 1
continue
active_index.append(lplb_count1)
if recordname in GVal.getPARA('label_raw_cache').keys():
label_raw = GVal.getPARA('label_raw_cache')[recordname]
# print('^^^' * 100)
else:
label_raw = np.loadtxt(label_file[0])
GVal.getPARA('label_raw_cache')[recordname] = label_raw
# print('&&&' * 100)
# GVal.setPARA('label_raw_cache', {recordname: label_raw})
subject_sample_length[lplb_count1] = len(label_raw)
# Label
label_signal[lplb_count1][0] = label_raw[:, 1]
# Label Confidence
label_signal[lplb_count1][1] = label_raw[:, 2]
lplb_count1 += 1
active_index += select_index[0]
return active_index, subject_sample_length, label_signal
def initializationProcess():
# Define the prefix, this is used for different computation environment.
localprefix = '/home/'
serverprefix = '/home/labcompute/'
# locipe local ip end
locipe = os.environ['SSH_CLIENT'][10:12]
print(locipe)
if locipe == '21':
# When start working with run_NFDA.sh (run in slave service)
GVal.setPARA('prefix_PARA', serverprefix)
username = os.environ["USERNAME"]
elif locipe == '45':
# When working with command: python3 NFDA_Launcher.py (run in master local)
GVal.setPARA('prefix_PARA', localprefix)
username = '******'
try:
os.remove(localprefix + 'GaoMY/EXECUTION/NFDA/code/tlog.txt')
except FileNotFoundError:
print(' ')
return GVal.getPARA('prefix_PARA'), username
def calculateFRAP(Z, y_val, screen):
# F - Fscore
# R - Recall
# A - Accuracy
# P - Precision
TP = len(np.nonzero(np.logical_and((Z == 1), (y_val == 1)))[0])
TN = len(np.nonzero(np.logical_and((Z == 0), (y_val == 0)))[0])
FP = len(np.nonzero(np.logical_and((Z == 1), (y_val == 0)))[0])
FN = len(np.nonzero(np.logical_and((Z == 0), (y_val == 1)))[0])
if TP + FP == 0:
P = -1
FA = -1
else:
P = TP / (TP + FP)
FA = 1 - P
if (TP + TN + FP + FN) == 0:
A = -1
else:
A = (TP + TN) / (TP + TN + FP + FN)
if TP + FN == 0:
R = -1
MA = -1
else:
R = TP / (TP + FN)
MA = 1 - R
# P = TP / (TP + FP)
# A = (TP + TN) / (TP + TN + FP + FN)
# R = TP / (TP + FN)
# MA = 1 - R
# FA = 1 - P
F1 = fScore(1, P, R)
beta = GVal.getPARA('beta_PARA')
F = fScore(beta, P, R)
FRAP = [P, A, R, MA, FA, F1, F]
if screen == 1:
print(('###[ P: ' + str(round(FRAP[0], 4)) +
' || A: ' + str(round(FRAP[1], 4)) +
' || R: ' + str(round(FRAP[2], 4)) +
' || MA: ' + str(round(FRAP[3], 4)) +
' || FA: ' + str(round(FRAP[4], 4)) +
' || F1: ' + str(round(FRAP[5], 4)) +
' || F' + str(beta) + ': ' + str(round(FRAP[6], 4)) + ']###'
))
return FRAP
def noiseFrameDilation(Y):
Nr_deprecate = GVal.getPARA('Nr_Frame_PARA')
NReinit = 3
noise_serial = copy.deepcopy(Y)
noise_serial_temp = copy.deepcopy(Y)
for iNR in range(NReinit - 1):
noise_serial_temp[iNR] = int(Y[iNR] == 0)
i_noise = NReinit - 1
while i_noise < len(noise_serial) - 1:
noise_serial_temp[i_noise] = int(Y[i_noise] == 0)
if sum(noise_serial_temp[i_noise - NReinit + 1:i_noise + 1]) == 0:
for j_noise in range(i_noise, min(i_noise + Nr_deprecate, len(Y) - 1)):
noise_serial[j_noise] = 9
i_noise += 1
return noise_serial
def voting(X_tra, y_tra, X_val, y_val, index_no, classifier_num):
#
classifier_list = GVal.getPARA('classifier_list_PARA')
# dVM[3400] = ['estimators', [21, 23, 25, 30, 31], [21, 23, 25, 30, 31]]
estims = []
for i in range(len(dVM[3400][2])):
clf_temp = (classifier_list[dVM[3400][2][i]][1], classifier_list[int(
str(dVM[3400][2][i])[0:2])][0](X_tra, y_tra, X_val, y_val,
index_no, dVM[3400][2][i])[0])
estims.append(clf_temp)
y_tra, X_tra, y_val, X_val, weights = dataRegulationSKL(
y_tra, X_tra, y_val, X_val, index_no)
clf = skemb.VotingClassifier(estimators=estims, voting=dVM[3401][2])
clf.fit(X_tra, y_tra)
return processLearning(clf, X_tra, y_tra, X_val, y_val)
def labelReading(select_index, data_file, path):
# Initialize several matrix
label_signal = cell2dmatlab_jsp([len(select_index), 2], 2, [])
subject_sample_length = np.zeros([len(select_index), 1])
active_index = []
GVal.initPARA('label_raw_cache', {'name': 'data'})
# Iplb LooPLaBel ,for the for loop
lplb_count1 = 0
for i in select_index:
recordname = data_file[i]
print(recordname)
# Loading of labels.
label_file = glob.glob(path['label_path'] + recordname + '*.txt')
# print(path['label_path'] + recordname + '*.txt')
if not label_file:
# return if no file in local database
lplb_count1 += 1
continue
active_index.append(lplb_count1)
# If the
if recordname in GVal.getPARA('label_raw_cache').keys():
label_raw = GVal.getPARA('label_raw_cache')[recordname]
else:
label_raw = np.loadtxt(label_file[0])
GVal.getPARA('label_raw_cache')[recordname] = label_raw
# Save the total frame length of the current recording
subject_sample_length[lplb_count1] = len(label_raw)
# Label
label_signal[lplb_count1][0] = label_raw[:, 1]
# Label Confidence
label_signal[lplb_count1][1] = label_raw[:, 2]
lplb_count1 += 1
active_index += select_index[0]
# Save for FINAL EVALUATION
GVal.setPARA('label_signal_PARA', label_signal)
return active_index, subject_sample_length, label_signal
def decisionTree(X_tra, y_tra, X_val, y_val, index_no, classifier_num):
y_tra, X_tra, y_val, X_val, weights = dataRegulationSKL(
y_tra, X_tra, y_val, X_val, index_no)
clf = sktree.DecisionTreeClassifier(criterion=dVM[3100][2],
splitter=dVM[3101][2],
max_depth=dVM[3102][2],
min_samples_split=dVM[3103][2],
min_samples_leaf=dVM[3104][2],
max_features=dVM[3106][2],
random_state=dVM[3107][2])
clf.fit(X_tra, y_tra, sample_weight=weights)
path = GVal.getPARA('path_PARA')
with open(path['fig_path'] + 'dtclf.dot', 'w') as f:
f = sktree.export_graphviz(clf, out_file=f, class_names=['0', '1'])
# sktree.export_graphviz(clf, out_file=path['fig_path'] + 'tree.dot')
# exit()
return processLearning(clf, X_tra, y_tra, X_val, y_val)
def calculateFRAP(Z, y_val):
TP = len(np.nonzero(np.logical_and((Z == 1), (y_val == 1)))[0])
TN = len(np.nonzero(np.logical_and((Z == 0), (y_val == 0)))[0])
FP = len(np.nonzero(np.logical_and((Z == 1), (y_val == 0)))[0])
FN = len(np.nonzero(np.logical_and((Z == 0), (y_val == 1)))[0])
P = TP / (TP + FP)
A = (TP + TN) / (TP + TN + FP + FN)
R = TP / (TP + FN)
MA = 1 - R
FA = 1 - P
F1 = fScore(1, P, R)
beta = GVal.getPARA('beta_PARA')
F = fScore(beta, P, R)
FRAP = [P, A, R, MA, FA, F1, F]
print(('###[ P: ' + str(round(FRAP[0], 4)) + ' || A: ' +
str(round(FRAP[1], 4)) + ' || R: ' + str(round(FRAP[2], 4)) +
' || MA: ' + str(round(FRAP[3], 4)) + ' || FA: ' +
str(round(FRAP[4], 4)) + ' || F1: ' + str(round(FRAP[5], 4)) +
' || F' + str(beta) + ': ' + str(round(FRAP[6], 4)) + ']###'))
return FRAP
def dataRegulationSKL(y_tra_in, X_tra_in, y_val_in, X_val_in, index_no):
# This function is working to deal with several data problems below:
# 1. Processing with the transitional frame (no1)
# 2. Processing wth label weights
kick_off_no1 = GVal.getPARA('kick_off_no1_PARA')
weights_on = GVal.getPARA('weights_on_PARA')
weight_list = GVal.getPARA('weight_list_PARA')
# Label weight issue
if weights_on:
weights = copy.deepcopy(y_tra_in)
for key in weight_list.keys():
weights[np.nonzero(weights == key)[0]] = weight_list[key]
print(weights)
else:
weights = np.ones(y_tra_in.shape)
# Transitional frame issue
kick_off_no1_switcher = {
0: ko1processor0,
1: ko1processor1,
2: ko1processor2,
3: ko1processor3,
4: ko1processor4,
5: ko1processor5,
6: ko1processor6,
7: ko1processor7,
8: ko1processor8
}
# Avoiding the overwrite problem when multiply excuting dataRegulationSKL
y_tra_temp = copy.deepcopy(y_tra_in)
X_tra_temp = copy.deepcopy(X_tra_in)
y_val_temp = copy.deepcopy(y_val_in)
X_val_temp = copy.deepcopy(X_val_in)
print('### Label 1 Processing Method: ' + GVal.getPARA('kick_off_no1_detail')[int(GVal.getPARA('kick_off_no1_PARA'))])
y_tra, X_tra, y_val, X_val, weights = kick_off_no1_switcher[int(GVal.getPARA('kick_off_no1_PARA'))](y_tra_temp, X_tra_temp, y_val_temp, X_val_temp, weights, index_no)
X_tra_res = copy.deepcopy(X_tra)
X_val_res = copy.deepcopy(X_val)
y_tra_res = copy.deepcopy(y_tra)
y_val_res = copy.deepcopy(y_val)
GVal.setPARA('y_tra_res_PARA', y_tra_res)
GVal.setPARA('y_val_res_PARA', y_val_res)
GVal.setPARA('X_tra_res_PARA', X_tra_res)
GVal.setPARA('X_val_res_PARA', X_val_res)
# Change the staged level 0,1,2,3 into a binary situation 0 and 1(1,2,3)
y_tra[np.nonzero(y_tra > 0)[0]] = 1
y_val[np.nonzero(y_val > 0)[0]] = 1
# Checking code, do not delete .
print('X train')
print(X_tra.shape)
print('y train')
print(y_tra.shape)
print(max(y_tra), min(y_tra))
print('X validation')
print(X_val.shape)
print('y validation')
print(y_val.shape)
print(max(y_val), min(y_val))
# print(len(np.nonzero(y_tra_res == 0)[0]))
# print(len(np.nonzero(y_tra_res == 1)[0]))
# print(len(np.nonzero(y_tra_res == 2)[0]))
# print(len(np.nonzero(y_tra_res == 3)[0]))
FLAG_temp = GVal.getPARA('FLAG_PARA')
pcode4fig = GVal.getPARA('process_code_PARA')
if FLAG_temp['hist_plotting_flag']:
ftsize = 16
hist_fea_list = GVal.getPARA('hist_fea_list_PARA')
path = GVal.getPARA('path_PARA')
nbins = 20
figcode = pcode4fig
h = plt.figure(num=figcode, figsize=(20, 30))
inu = 0
for ireal in hist_fea_list:
iserial = int(np.where(hist_fea_list == ireal)[0])
print(GVal.getPARA('online_fea_selectindex_PARA'))
inum = int(np.where(GVal.getPARA('online_fea_selectindex_PARA') == ireal)[0])
print('inu: ' + str(inu) + ' || ireal: ' + str(ireal) + '|| iserial: ' + str(iserial) + '|| inum: ' + str(inum))
x = 0.03
y = 0.89
a = -0.02
le = 0.19
lfig = 0.17
k = -0.085
kfig = 0.065
posx = [x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le,
x, x + le, x + 2 * le, x + 3 * le, x + 4 * le]
posy = [y, y, y, y, y, y + k, y + k, y + k, y + k, y + k,
y + 2 * k + a, y + 2 * k + a, y + 2 * k + a, y + 2 * k + a, y + 2 * k + a,
y + 3 * k + a, y + 3 * k + a, y + 3 * k + a, y + 3 * k + a, y + 3 * k + a,
y + 4 * k + 2 * a, y + 4 * k + 2 * a, y + 4 * k + 2 * a, y + 4 * k + 2 * a, y + 4 * k + 2 * a,
y + 5 * k + 2 * a, y + 5 * k + 2 * a, y + 5 * k + 2 * a, y + 5 * k + 2 * a, y + 5 * k + 2 * a,
y + 6 * k + 3 * a, y + 6 * k + 3 * a, y + 6 * k + 3 * a, y + 6 * k + 3 * a, y + 6 * k + 3 * a,
y + 7 * k + 3 * a, y + 7 * k + 3 * a, y + 7 * k + 3 * a, y + 7 * k + 3 * a, y + 7 * k + 3 * a,
y + 8 * k + 4 * a, y + 8 * k + 4 * a, y + 8 * k + 4 * a, y + 8 * k + 4 * a, y + 8 * k + 4 * a,
y + 9 * k + 4 * a, y + 9 * k + 4 * a, y + 9 * k + 4 * a, y + 9 * k + 4 * a, y + 9 * k + 4 * a]
# print(len(posx))
# print(len(posy))
inum1 = int(10 * np.floor((inu) / 5) + (iserial) % 5)
# print(inu, inum1)
# print(posx[inum1], posy[inum1])
plt.axes([posx[inum1], posy[inum1], lfig, kfig])
sns.distplot(X_tra_res[np.nonzero(y_tra_res > 1)[0], inum], bins=nbins, rug=False, label='posi [23]')
sns.distplot(X_tra_res[np.nonzero(y_tra_res == 0)[0], inum], bins=nbins, rug=False, label='nega [0]')
sns.distplot(X_tra_res[np.nonzero(y_tra_res == 1)[0], inum], bins=nbins, rug=False, label='tranz [1]')
# plt.legend .....
plt.title('[ Feature No. ' + str(ireal) + ' ] Trainning Set ')
plt.legend()
inum2 = int(10 * np.floor((inu) / 5) + (iserial) % 5 + 5)
# print(inu, inum2)
plt.axes([posx[inum2], posy[inum2], lfig, kfig])
sns.distplot(X_val_res[np.nonzero(y_val_res > 1)[0], inum], bins=nbins, rug=False, label='posi[23]')
sns.distplot(X_val_res[np.nonzero(y_val_res == 0)[0], inum], bins=nbins, rug=False, label='nega[0]')
sns.distplot(X_val_res[np.nonzero(y_val_res == 1)[0], inum], bins=nbins, rug=False, label='tranz [1]')
# print(GVal.getPARA('online_fea_name_PARA')[i][0])
plt.title('[ ' + GVal.getPARA('online_fea_name_PARA')[ireal][0] + ' ] Testing Set ', fontproperties=zhfont, fontsize=ftsize)
if inu == 0:
plt.legend()
inu += 1
plt.show()
plt.savefig((path['fig_path'] + 'Histo_Fea_Pic' + str(figcode) + '.png'))
print('Histo Feature Picture' + str(figcode) + 'Saved!')
else:
print('### No hist feature plotting... Skip')
return y_tra, X_tra, y_val, X_val, weights
def mainProcesser(process_code):
# The decoder translate the process_code into different parameter to be set.
select_index, classifier_num = processCodeDecoder(process_code)
# Prepare some information
online_fea_selectindex = GVal.getPARA('online_fea_selectindex_PARA')
noise_label_index = GVal.getPARA('noise_label_index_PARA')
####################################
# [TODO] Raw Data Reading methods
# FUNCTION rawDataReading()
####################################
# [TODO] New preprocessing methods
# FUNCTION raw_dev_fea = preProcessing()
# Output raw_dev_fea should then become input of the Function featurePreparation()
if FLAG['data_prepare_flag'] and GVal.getPARA('firstTimeConstruction'):
# Load the label information from local database
active_index, subject_sample_length, label_signal = labelReading(
select_index, data_file, path)
# Load and prepare the feature from local database
online_fea_signal, dev_fea_signal, noise_label_signal = featurePreparation(
select_index, online_fea_selectindex, noise_label_index,
active_index, data_file)
# Construct all the information, make a standard format. [label_all] & [online_fea_all]
label_all, online_fea_all = dataConstruction(
select_index, online_fea_selectindex, subject_sample_length,
label_signal, online_fea_signal, noise_label_signal, path,
FLAG['save_flag'])
GVal.setPARA('firstTimeConstruction', 0)
else:
print('### Skip the data construction, load from the database')
print('### The loading data directory is: ' + path['pdata_path'])
label_all = np.loadtxt((path['pdata_path'] + 'label_all.txt'))
online_fea_all = np.loadtxt(
(path['pdata_path'] + 'online_fea_all.txt'))
X_rawraw = np.transpose(online_fea_all)
Y_raw = np.transpose(label_all)
print('######## [Data Preparation Accomplished! ] ########')
print('### Label Raw Matrix Size: ' + str(Y_raw.shape))
print('### Feature Raw Matrix Size: ' + str(X_rawraw.shape))
# Until here, all the developed label and features are prepared in a certain format.
# for one single data sample, get the label by:
# snum: Sample NUM || fnum: Feature NUM
# label_all[0,snum] : label
# label_all[1,snum] : confidence
# online_fea_all[fnum,snum] : difference features.
####################################
# [TODO DONE!] Compare with Y. Do the validation . Try leave-one-out, or K-fold
X_train_rawraw, X_validation_raw, y_train_raw, y_validation_raw, N = datasetSeparation(
X_rawraw, Y_raw, GVal.getPARA('split_type_num'),
GVal.getPARA('split_type_para'))
# Sample: X_train[n_fold][fnum,snum]
classifier_frame = {3: sklearnTrainer, 2: sklearnTrainer, 1: kerasTrainer}
# Loop working when n fold is working.
for N_lplb_count in np.arange(N):
print(' ')
print('#' * 32)
print('######## [Fold (#' + str(N_lplb_count + 1) + '/' + str(N) +
') ] ########')
# Prepare the F, Y
if FLAG['label_process_flag']:
# label processor targeting on noconfident frame / noise frame / noblink frame 'N3 frame'
# Extendable function strucutre. For any customized label processing tasks.
X_train_raw, y_train, X_validation_rawS, y_validation_rawS = labelProcessor(
X_train_rawraw[N_lplb_count], y_train_raw[N_lplb_count],
X_validation_raw[N_lplb_count], y_validation_raw[N_lplb_count])
else:
X_train_raw = copy.deepcopy(X_train_rawraw[N_lplb_count])
y_train = copy.deepcopy(y_train_raw[N_lplb_count])
X_validation_rawS = copy.deepcopy(X_validation_raw[N_lplb_count])
y_validation_rawS = copy.deepcopy(y_validation_raw[N_lplb_count])
# # # Feature Plotting [N/A]
# if FLAG['fea_plotting_flag']:
# featurePlotting(Y_train, X_train_raw, process_code)
# else:
# print('### No plotting, exit feature plotting...')
####################################
# [TODO IN PROGRESS] Add the feature selection Method
if FLAG['fea_selection_flag']:
# feaSelection is doing the feature selection only on trainning set, return the fea selection result
X_train, feaS_mdl = feaSelection(X_train_raw)
# According the fea selection result, preprocess validation set in corresponding to the trainning set.
X_validation = feaS_mdl.fit_transform(X_validation_rawS)
else:
X_train = copy.deepcopy(X_train_raw)
X_validation = copy.deepcopy(X_validation_rawS)
####################################
# [TODO] Feature Regulation
# Three types of features:
# 1 -- Continuous
# 2 -- Discrete (Categorical)
# 3 -- Discrete Binary (0 and 1)
y_validation = copy.deepcopy(y_validation_rawS)
####################################
# [TODO IN PROGRESS] Data balancing method!
# Data balance only affect on training set.
# The method is solving the unbalance problem.
# The function is extendable, customized data balancing method can be applied.
if FLAG['data_balance_flag']:
X_train, y_train = dataBalance(X_train, y_train)
res_temp = cell2dmatlab_jsp([1, 5], 2, [])
# [TODO IN PROGRESS] Classifier Designing
res_temp[0][0] = process_code
res_temp[0][1:4] = classifier_frame[int(str(classifier_num)[0])](
classifier_num, X_train, y_train, X_validation, y_validation, path)
# Sample: res[fold number][infoserial]
# infoserial : 1- model object [1: num, 2:name, 3:mdl obejct] 2- Training Score 3- FRAP
resCalMethodList = {'median': np.median, 'mean': np.mean}
if N == 1:
res = res_temp
else:
if N_lplb_count == 0:
res_mid = res_temp
elif N_lplb_count < N - 1:
res_mid += res_temp
elif N_lplb_count == N - 1:
res_mid += res_temp
res = res_temp
for ires in range(len(res[0][3])):
temptemp = []
for iN in range(N):
# if np.logical_and(res_mid[iN][3][ires] != 0.0, res_mid[iN][3][ires] != 1.0):
if res_mid[iN][3][ires] not in [0.0, 1.0, -1.0]:
temptemp.append(res_mid[iN][3][ires])
res[0][3][ires] = resCalMethodList[GVal.getPARA(
'resCalMethod_PARA')](temptemp)
if FLAG['pairDristributionPlotting_flag']:
FRAP = res_temp[0][3]
pairDistPlotting(FRAP, process_code)
if FLAG['resultDristributionPlotting_flag']:
FRAP = res_temp[0][3]
resultPlotting(FRAP, process_code)
else:
print('### No plotting, exit feature plotting...')
time.sleep(0.001)
return res, N
'label_path': (path_prefix + 'public/backend_data/Label/'),
'online_fea_path': (path_prefix + 'public/backend_data/online_new/'),
'test_fea_path': (path_prefix + 'public/backend_data/test_fea/'),
'parent_path': (path_prefix + username + '/EXECUTION/NFDA/code/')
}
# Define the several sub path
# work path, in which all the processing code are placed
path['work_path'] = (path['parent_path'] + 'python_code/')
# fig path, to save the result figs
path['fig_path'] = (path['parent_path'] + 'python_code/fig/')
# pdata_path, to save several types of processing data, (temporial data for accelerating the procedure)
path['pdata_path'] = (path['parent_path'] + 'python_code/data/')
# Global Statement of [path]
GVal.setPARA('path_PARA', path)
# Change the current path if necessary.
os.chdir(path['work_path'])
# RawData folder list reading.
data_file_temp = sorted(os.listdir(path['online_fea_path']))
data_file = data_file_temp
knnn = 0
for i in data_file_temp:
print(str(knnn) + ' || ' + i)
data_file[knnn] = data_file_temp[knnn][:-4]
knnn += 1
# exit()
# Processing the Control Panel! All the important configuration are pre-defined in control panel.
# Refer the file controlPanel_NFDA_J.py for more details.
return res, N
if __name__ == "__main__":
start = clock()
path_prefix = initializationProcess()
path = {
'data_path': (path_prefix + 'public/EEG_RawData/'),
'label_path': (path_prefix + 'GaoMY/EXECUTION/NFDA/Label/'),
'parent_path': (path_prefix + 'GaoMY/EXECUTION/NFDA/code/'),
'online_fea_path': (path_prefix + 'GaoMY/EXECUTION/NFDA/online_new/')
}
path['work_path'] = (path['parent_path'] + 'python_code/')
path['pdata_path'] = (path['parent_path'] + 'data/')
GVal.setPARA('path_PARA', path)
os.chdir(path['work_path'])
# RawData Reading.
data_file = sorted(os.listdir(path['data_path']))
FLAG, process_code_pack = controlPanel()
# ProcessingCodeLoop
L_lplb_count = 0
L = len(process_code_pack)
for process_code in process_code_pack:
GVal.setPARA('process_code_PARA', process_code)
res_singleLOOP, N = mainProcesser(process_code)
if L_lplb_count == 0:
res = res_singleLOOP
def mainProcesser(process_code):
select_index, classifier_num = processCodeDecoder(process_code, 1)
online_fea_selectindex = GVal.getPARA('online_fea_selectindex_PARA')
noise_label_index = GVal.getPARA('noise_label_index_PARA')
################################################################################
############## [ Engine START! ] ###################################################
################################################################################
####################################
# [TODO] Raw Data Reading methods
# FUNCTION rawDataReading_NFDA()
####################################
# [TODO] New preprocessing methods
# FUNCTION raw_dev_fea = preProcessing_NFDA()
# Output raw_dev_fea should then become input of the Function featurePreparation_NFDA()
if FLAG['data_prepare_flag'] and GVal.getPARA('firstTimeConstruction'):
active_index, subject_sample_length, label_signal = labelReading_NFDA(select_index, data_file, path)
online_fea_signal, dev_fea_signal, noise_label_signal = featurePreparation_NFDA(select_index, online_fea_selectindex, noise_label_index, active_index, data_file)
label_all, online_fea_all = dataConstruction_NFDA(select_index, online_fea_selectindex, subject_sample_length, label_signal, online_fea_signal, noise_label_signal, path, FLAG['save_flag'])
GVal.setPARA('firstTimeConstruction', 0)
else:
print('### Skip the data construction, load from the database')
print('### The loading data directory is: ' + path['pdata_path'])
label_all = np.loadtxt((path['pdata_path'] + 'label_all.txt'))
online_fea_all = np.loadtxt((path['pdata_path'] + 'online_fea_all.txt'))
X_rawraw = np.transpose(online_fea_all)
Y_raw = np.transpose(label_all)
print('######## [Data Preparation Accomplished! ] ########')
print('### Label Raw Matrix Size: ' + str(Y_raw.shape))
print('### Feature Raw Matrix Size: ' + str(X_rawraw.shape))
# Until here, all the developed label and features are prepared in a certain format.
# for one single data sample, get the label by:
# snum: Sample NUM || fnum: Feature NUM
# label_all[0,snum] : label
# label_all[1,snum] : confidence
# online_fea_all[fnum,snum] : difference features.
# Feature Plotting [N/A]
if FLAG['plotting_flag']:
featurePlotting_NFDA(label_all, online_fea_all)
else:
print('### No plotting, exit feature plotting...')
# Prepare the F, Y
# X_raw[snum, fnum]
# Y[snum,:]
if FLAG['label_process_flag']:
X_raw, Y = labelProcessor(X_rawraw, Y_raw)
else:
X_raw = copy.deepcopy(X_rawraw)
Y = copy.deepcopy(Y_raw)
####################################
# [TODO IN PROGRESS] Add the feature selection Method
if FLAG['fea_selection_flag']:
X = feaSelection(X_raw)
else:
X = copy.deepcopy(X_raw)
####################################
# [TODO] Feature Regulation
# Three types of features:
# 1 -- Continuous
# 2 -- Discrete (Categorical)
# 3 -- Discrete Binary (0 and 1)
####################################
# [TODO DONE!] Compare with Y. Do the validation . Try leave-one-out, or K-fold
X_train, X_validation, y_train, y_validation, N = datasetSeparation(X, Y, GVal.getPARA('split_type_num'), GVal.getPARA('split_type_para'))
# Sample: X_train[n_fold][fnum,snum]
classifier_frame = {
3: sklearnTrainer,
2: sklearnTrainer,
1: kerasTrainer
}
# Loop working when n fold is working.
for N_lplb_count in np.arange(N):
####################################
# [TODO IN PROGRESS] Data balancing method!
# [IMPORTANT!] Data balance should be after the separation and only affect on training set!
if FLAG['data_balance_flag']:
X_train[N_lplb_count], y_train[N_lplb_count] = dataBalance(X_train[N_lplb_count], y_train[N_lplb_count])
res_temp = cell2dmatlab_jsp([1, 4], 2, [])
# [TODO IN PROGRESS] Classifier Designing
# Try different classifiers, Get CY
print(' ')
print('#' * 35)
print('######## [Fold (#' + str(N_lplb_count + 1) + '/' + str(N) + ') ] ########')
print('#' * 35)
res_temp[0][0] = process_code
res_temp[0][1:4] = classifier_frame[int(str(classifier_num)[0])](classifier_num, X_train[N_lplb_count], y_train[N_lplb_count], X_validation[N_lplb_count], y_validation[N_lplb_count], path)
# Sample: res[fold number][infoserial]
# infoserial : 0- model object 1- Training Score 2- FRAP
if N_lplb_count == 0:
res = res_temp
else:
res += res_temp
time.sleep(0.001)
return res, N
