def main_EXP_G_VULN_GEO_LOCATION_train_single_ANN_remapping():
read_command_line_options()
thismodule = sys.modules[__name__]
EXP_G_VULN_GEO_LOCATION_FOLDER = "/home/comete/mromanel/MILES_EXP/EXP_GEO_LOCATION_QIF_LIB_SETTING/"
utilities.createFolder(EXP_G_VULN_GEO_LOCATION_FOLDER)
RESULT_FOLDER = EXP_G_VULN_GEO_LOCATION_FOLDER + "RESULT_FOLDER_REMAPPING/"
utilities.createFolder(RESULT_FOLDER)
result_folder = RESULT_FOLDER + MODEL_NAME + "/"
utilities.createFolder(result_folder)
result_folder = result_folder + str(
TRAINING_SIZE) + "_training_size_and_" + str(
VALIDATION_SIZE) + "_validation_size_iteration_" + str(
TRAINING_ITERATION) + "/"
utilities.createFolder(result_folder)
DATA_FOLDER = EXP_G_VULN_GEO_LOCATION_FOLDER + "DATA_FOLDER_AFTER_OUR_PREPROCESSING/"
ANN_data_folder = DATA_FOLDER + str(TRAINING_SIZE) + "_training_and_" + str(
VALIDATION_SIZE) + "_validation_store_folder_train_iteration_" + str(
TRAINING_ITERATION)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% load datasets %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
print "\n\n\nDATA ARE LOADED FROM ", ANN_data_folder, "\n\n\n"
log_file = open(result_folder + "/log_file.txt", "wa")
log_file.write("\n\n\nDATA ARE LOADED FROM " + ANN_data_folder + "\n\n\n")
log_file.close()
training_set = pn.read_pickle(path=ANN_data_folder + "/training_set.pkl")
O_train = training_set.values[:, 0]
S_train = training_set.values[:, 1]
Z_train = training_set.values[:, 2]
# Z_train_enc = preprocess.array_one_hot_encoder(supervision_=Z_train)
"""Z_train_list = []
O_train_unq = np.unique(O_train)
for otu in O_train_unq:
idx = np.where(O_train == otu)[0]
tmp = np.mean(Z_train_enc[idx, :], axis=0)
idx_max = np.argmax(tmp)
for r in range(len(tmp)):
if r != idx_max:
tmp[r] = 0.
else:
tmp[r] = 1.
Z_train_list.append(tmp)
Z_train_final_list = []
for ot in O_train:
idx = np.where(O_train_unq == ot)[0]
for el in idx:
Z_train_final_list.append(Z_train_list[el])
# Z_train = np.array(Z_train_final_list).reshape((Z_train_enc.shape[0], Z_train_enc.shape[1]))
Z_train = np.array(Z_train_list).reshape((len(O_train_unq), Z_train_enc.shape[1]))"""
val_set = pn.read_pickle(path=ANN_data_folder + "/validation_set.pkl")
O_val = val_set.values[:, 0]
S_val = val_set.values[:, 1]
Z_val = val_set.values[:, 2]
# Z_val_enc = preprocess.array_one_hot_encoder(supervision_=Z_val)
"""Z_val_list = []
O_val_unq = np.unique(O_val)
for otu in O_val_unq:
idx = np.where(O_val == otu)[0]
tmp = np.mean(Z_val_enc[idx, :], axis=0)
idx_max = np.argmax(tmp)
for r in range(len(tmp)):
if r != idx_max:
tmp[r] = 0.
else:
tmp[r] = 1.
Z_val_list.append(tmp)
Z_val_final_list = []
for ot in O_val:
idx = np.where(O_val_unq == ot)[0]
for el in idx:
Z_val_final_list.append(Z_val_list[el])
# Z_val = np.array(Z_val_final_list).reshape((Z_val_enc.shape[0], Z_val_enc.shape[1]))
Z_val = np.array(Z_val_list).reshape((len(O_val_unq), Z_val_enc.shape[1]))"""
"""O_train = preprocess.scaler_between_minus_one_and_one(column=O_train, min_column=MIN_OBSERVABLE,
max_column=MAX_OBSERVABLE)
O_val = preprocess.scaler_between_minus_one_and_one(column=O_val, min_column=MIN_OBSERVABLE,
max_column=MAX_OBSERVABLE)"""
min_max_scaler = preprocessing.MinMaxScaler()
O_train = O_train.reshape(-1, 1)
O_train = min_max_scaler.fit_transform(O_train)
O_val = O_val.reshape(-1, 1)
O_val = min_max_scaler.transform(O_val)
Z_train_enc = one_hot_enc(y=Z_train, num_classes=N_CLASSES)
Z_val_enc = one_hot_enc(y=Z_val, num_classes=N_CLASSES)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ANN: instantiate, train, evaluate %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if len(O_train.shape) == 1:
input_x_dimension = 1
else:
input_x_dimension = O_train.shape[1]
if thismodule.BATCH_SIZE == None:
thismodule.BATCH_SIZE = O_train.shape[0]
secrets_classifier_manager = secrets_classifier.ClassifierNetworkManager(
number_of_classes=Z_train_enc.shape[1],
learning_rate=LEARNING_RATE,
hidden_layers_card=HIDDEN_LAYERS_CARD,
hidden_neurons_card=HIDDEN_NEAURONS_CARD,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
id_gpu=ID_GPU,
perc_gpu=PERC_GPU,
input_x_dimension=input_x_dimension)
secrets_classifier_manager.train_classifier_net(
training_set=O_train,
training_supervision=Z_train_enc,
validation_set=O_val,
validation_supervision=Z_val_enc,
results_folder=result_folder)
def main_EXP_G_VULN_PSW_train_single_ANN_remapping():
read_command_line_options()
thismodule = sys.modules[__name__]
EXP_PSW_FOLDER = "/home/comete/mromanel/MILES_EXP/EXP_PSW/"
utilities.createFolder(EXP_PSW_FOLDER)
RESULT_FOLDER = EXP_PSW_FOLDER + "RESULT_FOLDER_REMAPPING/"
utilities.createFolder(RESULT_FOLDER)
result_folder = RESULT_FOLDER + MODEL_NAME + "/"
utilities.createFolder(result_folder)
result_folder = result_folder + str(
TRAINING_SIZE) + "_training_size_and_" + str(
VALIDATION_SIZE) + "_validation_size_iteration_" + str(
TRAINING_ITERATION) + "/"
utilities.createFolder(result_folder)
DATA_FOLDER = EXP_PSW_FOLDER + "DATA_FOLDER_AFTER_OUR_PREPROCESSING/"
ANN_data_folder = DATA_FOLDER + str(TRAINING_SIZE) + "_training_and_" + str(
VALIDATION_SIZE) + "_validation_store_folder_train_iteration_" + str(
TRAINING_ITERATION)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% load datasets %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
print "\n\n\nDATA ARE LOADED FROM ", ANN_data_folder, "\n\n\n"
log_file = open(result_folder + "/log_file.txt", "wa")
log_file.write("\n\n\nDATA ARE LOADED FROM " + ANN_data_folder + "\n\n\n")
log_file.close()
training_set = pn.read_pickle(path=ANN_data_folder + "/training_set.pkl")
O_train = training_set[:, 0]
Z_train = training_set[:, 1]
Z_train_enc = to_categorical(y=Z_train, num_classes=NUM_CLASSES)
val_set = pn.read_pickle(path=ANN_data_folder + "/training_set.pkl")
O_val = val_set[:, 0]
Z_val = val_set[:, 1]
Z_val_enc = to_categorical(y=Z_val, num_classes=NUM_CLASSES)
"""O_train = preprocess.scaler_between_minus_one_and_one(column=O_train, min_column=MIN_OBSERVABLE,
max_column=MAX_OBSERVABLE)
O_val = preprocess.scaler_between_minus_one_and_one(column=O_val, min_column=MIN_OBSERVABLE,
max_column=MAX_OBSERVABLE)"""
min_max_scaler = preprocessing.MinMaxScaler()
O_train = O_train.reshape(-1, 1)
O_train = min_max_scaler.fit_transform(O_train)
O_val = O_val.reshape(-1, 1)
O_val = min_max_scaler.transform(O_val)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ANN: instantiate, train, evaluate %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if len(O_train.shape) == 1:
input_x_dimension = 1
else:
input_x_dimension = O_train.shape[1]
if thismodule.BATCH_SIZE is None:
thismodule.BATCH_SIZE = O_train.shape[0]
secrets_classifier_manager = secrets_classifier.ClassifierNetworkManager(
number_of_classes=Z_train_enc.shape[1],
learning_rate=LEARNING_RATE,
hidden_layers_card=HIDDEN_LAYERS_CARD,
hidden_neurons_card=HIDDEN_NEAURONS_CARD,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
id_gpu=ID_GPU,
perc_gpu=PERC_GPU,
input_x_dimension=input_x_dimension)
secrets_classifier_manager.train_classifier_net(
training_set=O_train,
training_supervision=Z_train_enc,
validation_set=O_val,
validation_supervision=Z_val_enc,
results_folder=result_folder)
def main_BIS_EXP_G_VULN_DP_train_single_ANN_remapping():
read_command_line_options()
thismodule = sys.modules[__name__]
BIS_EXP_G_VULN_DP_FOLDER = "/home/comete/mromanel/MILES_EXP/BIS_EXP_G_VULN_DP_FOLDER/"
utilities.createFolder(BIS_EXP_G_VULN_DP_FOLDER)
RESULT_FOLDER = BIS_EXP_G_VULN_DP_FOLDER + "RESULT_FOLDER_REMAPPING/"
utilities.createFolder(RESULT_FOLDER)
result_folder = RESULT_FOLDER + MODEL_NAME + "/"
utilities.createFolder(result_folder)
result_folder = result_folder + str(
TRAINING_SIZE) + "_training_size_and_" + str(
VALIDATION_SIZE) + "_validation_size_iteration_" + str(
TRAINING_ITERATION) + "/"
utilities.createFolder(result_folder)
DATA_FOLDER = BIS_EXP_G_VULN_DP_FOLDER + "DATA_FOLDER/"
ANN_data_folder = DATA_FOLDER + str(TRAINING_SIZE) + "_training_and_" + str(
VALIDATION_SIZE) + "_validation_store_folder_train_iteration_" + str(
TRAINING_ITERATION)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% load datasets %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
print "\n\n\nDATA ARE LOADED FROM ", ANN_data_folder, "\n\n\n"
log_file = open(result_folder + "/log_file.txt", "wa")
log_file.write("\n\n\nDATA ARE LOADED FROM " + ANN_data_folder + "\n\n\n")
log_file.close()
training_set = pn.read_pickle(path=ANN_data_folder + "/training_set.pkl")
O_train = training_set[:, 0:training_set.shape[1] - 2]
print O_train.shape
S_train = training_set[:, -2]
Z_train = training_set[:, -1]
Z_train_enc = to_categorical(y=Z_train, num_classes=NUM_CLASSES)
val_set = pn.read_pickle(path=ANN_data_folder + "/validation_set.pkl")
O_val = val_set[:, 0:val_set.shape[1] - 2]
S_val = val_set[:, -2]
Z_val = val_set[:, -1]
Z_val_enc = to_categorical(y=Z_val, num_classes=NUM_CLASSES)
min_ = np.min(O_train)
# print min_
max_ = np.max(O_train)
# print max_
O_train = preprocess.scaler_zero_one_all_cols(data=O_train,
min_=min_,
max_=max_)
O_val = preprocess.scaler_zero_one_all_cols(data=O_val,
min_=min_,
max_=max_)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ANN: instantiate, train, evaluate %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if len(O_train.shape) == 1:
input_x_dimension = 1
else:
input_x_dimension = O_train.shape[1]
if thismodule.BATCH_SIZE is None:
thismodule.BATCH_SIZE = O_train.shape[0]
secrets_classifier_manager = secrets_classifier.ClassifierNetworkManager(
number_of_classes=Z_train_enc.shape[1],
learning_rate=LEARNING_RATE,
hidden_layers_card=HIDDEN_LAYERS_CARD,
hidden_neurons_card=HIDDEN_NEAURONS_CARD,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
id_gpu=ID_GPU,
perc_gpu=PERC_GPU,
input_x_dimension=input_x_dimension)
secrets_classifier_manager.train_classifier_net(
training_set=O_train,
training_supervision=Z_train_enc,
validation_set=O_val,
validation_supervision=Z_val_enc,
results_folder=result_folder)