print("STEP 3 : DONE")
print("---" * 100)
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 4 : CREATE DATA PROBA AND CLASSIFY")
print()
print("NEW DATA: " + str(args.create_new_data_for_classifier))
print()
generator_data = Genrator_data_prob_classifier(
args, nn_model_ref.net, path_save_model, rng, creator_data_binary,
device, get_masks_gen.masks, nn_model_ref)
generator_data.create_data_g(table_of_truth)
#EVALUATE GOHR NN ON NEW DATASET
nn_model_ref.X_train_nn_binaire = generator_data.X_bin_train
nn_model_ref.X_val_nn_binaire = generator_data.X_bin_val
nn_model_ref.Y_train_nn_binaire = generator_data.Y_create_proba_train
nn_model_ref.Y_val_nn_binaire = generator_data.Y_create_proba_val
if args.eval_nn_ref:
nn_model_ref.eval_all(["train", "val"])
all_clfs = All_classifier(args, path_save_model, generator_data,
get_masks_gen, nn_model_ref, table_of_truth,
cpt)
#all_clfs.X_train_proba = np.concatenate((all_clfs.X_train_proba, X_feat_temp), axis = 1)
#all_clfs.X_eval_proba = np.concatenate((all_clfs.X_eval_proba, X_feat_temp_val), axis = 1)
all_clfs.classify_all()
if args.quality_of_masks:
qm = Quality_masks(args, path_save_model, generator_data,
get_masks_gen, nn_model_ref, table_of_truth,
print("---" * 100)
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 4 : CREATE DATA PROBA AND CLASSIFY")
print()
print("NEW DATA: "+ str(args.create_new_data_for_classifier))
print()
generator_data = Genrator_data_prob_classifier(args, nn_model_ref.net, path_save_model, rng, creator_data_binary, device, get_masks_gen.masks, nn_model_ref)
generator_data.create_data_g(table_of_truth)
args.inputs_type = ["ctdata0l", "ctdata0r", "ctdata1l", "ctdata1r"]
nn_model_ref.load_nn()
#EVALUATE GOHR NN ON NEW DATASET
liste_inputs = creator_data_binary.convert_data_inputs(args, generator_data.c0l_create_proba_train, generator_data.c0r_create_proba_train, generator_data.c1l_create_proba_train, generator_data.c1r_create_proba_train)
nn_model_ref.X_train_nn_binaire = creator_data_binary.convert_to_binary(liste_inputs);
liste_inputs = creator_data_binary.convert_data_inputs(args, generator_data.c0l_create_proba_val, generator_data.c0r_create_proba_val, generator_data.c1l_create_proba_val, generator_data.c1r_create_proba_val)
nn_model_ref.X_val_nn_binaire = creator_data_binary.convert_to_binary(liste_inputs);
nn_model_ref.Y_train_nn_binaire = generator_data.Y_create_proba_train
nn_model_ref.Y_val_nn_binaire = generator_data.Y_create_proba_val
if args.eval_nn_ref:
nn_model_ref.eval_all(["train", "val"])
all_clfs = All_classifier(args, path_save_model, generator_data, get_masks_gen, nn_model_ref, table_of_truth)
#all_clfs.X_train_proba = np.concatenate((all_clfs.X_train_proba, X_feat_temp), axis = 1)
#all_clfs.X_eval_proba = np.concatenate((all_clfs.X_eval_proba, X_feat_temp_val), axis = 1)
all_clfs.classify_all()
if args.quality_of_masks:
qm = Quality_masks(args, path_save_model, generator_data, get_masks_gen, nn_model_ref, table_of_truth, all_clfs)
qm.start_all()
else:
qm = None
#data = data_train #np.array(nn_model_ref2.intermediaires[phase]).astype(np.uint8).reshape(num1 * nn_model_ref2.batch_size, -1)
#data2 = scaler1.fit_transform(data)
nn_model_ref2.all_intermediaire = data_train
#nn_model_ref2.outputs_proba_train = np.array(nn_model_ref2.outputs_proba[phase]).astype(np.float16).reshape(num1 * nn_model_ref2.batch_size, -1)
#nn_model_ref2.outputs_pred_train = np.array(nn_model_ref2.outputs_pred[phase]).astype(np.float16).reshape(num1 * nn_model_ref2.batch_size, -1)
#if not nn_model_ref2.args.retrain_nn_ref:
#del nn_model_ref2.all_intermediaire, data_train
else:
#scaler2 = StandardScaler()
#data = data_val
#data = np.array(nn_model_ref2.intermediaires[phase]).astype(np.uint8).reshape(num1 * nn_model_ref2.batch_size, -1)
#data2 = scaler2.fit_transform(data)
nn_model_ref2.all_intermediaire_val = data_val
#nn_model_ref2.outputs_proba_val = np.array(nn_model_ref2.outputs_proba[phase]).astype(np.float16).reshape(num2 * nn_model_ref2.batch_size, -1)
#nn_model_ref2.outputs_pred_val = np.array(nn_model_ref2.outputs_pred[phase]).astype(np.float16).reshape(
# num2 * nn_model_ref2.batch_size, -1)
#if not nn_model_ref2.args.retrain_nn_ref:
#del nn_model_ref2.all_intermediaire_val, data_val
del nn_model_ref2.dataloaders[phase]
for i in range(4):
print(data_train_X[data_train_preds == i].shape)
print(data_train_labels[data_train_preds == i].shape)
nn_model_ref2.X_train_nn_binaire = data_train_X[data_train_preds == i]
nn_model_ref2.X_val_nn_binaire = data_val_X[data_val_preds == i]
nn_model_ref2.Y_train_nn_binaire = data_train_labels[data_train_preds == i]
nn_model_ref2.Y_val_nn_binaire = data_val_labels[data_val_preds == i]
nn_model_ref2.train_general(name_input)
nn_model_ref2.choose_model()
vals, counts = np.unique(X_train, axis=1, return_counts=True)
print(vals.shape, counts.shape)
sv = dict(zip(vals, counts / num_samples))
print(sv)
"""
Y_eval_proba = nn_model_ref.Y_val_nn_binaire
Y_train_proba = nn_model_ref.Y_train_nn_binaire
#net = AE_binarize(args, X_train_f.shape[1], h1= 250).to(device)
net = NN_linear(args, X_train_f.shape[1]).to(device)
nn_model_ref.net = net
nn_model_ref.X_train_nn_binaire = X_train_f
nn_model_ref.X_val_nn_binaire = X_val_f
#nn_model_ref.Y_train_nn_binaire = X_train_f
#nn_model_ref.Y_val_nn_binaire = X_val_f
"""
args.load_nn_path = "./results/create_synth_masks_v2/speck/5/ctdata0l^ctdata1l_ctdata0r^ctdata1r^ctdata0l^ctdata1l_ctdata0l^ctdata0r_ctdata1l^ctdata1r/2020_07_21_17_26_59_603174/0.9966710913033485_bestacc.pth"
nn_model_ref.net.load_state_dict(torch.load(args.load_nn_path,
map_location=device)['state_dict'], strict=False)
nn_model_ref.net.to(device)
nn_model_ref.net.eval()
"""
nn_model_ref.train_from_scractch("AE")
for global_sparsity in [0, 0.2, 0.4]:
print(global_sparsity)
flag2 = True
acc_retain = []
del table_of_truth.c0l_create_ToT, table_of_truth.c0r_create_ToT
del table_of_truth.c1l_create_ToT, table_of_truth.c1r_create_ToT
print("STEP 3 : DONE")
print("---" * 100)
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 4 : CREATE DATA PROBA AND CLASSIFY")
print()
print("NEW DATA: "+ str(args.create_new_data_for_classifier))
print()
generator_data = Genrator_data_prob_classifier(args, nn_model_ref.net, path_save_model, rng, creator_data_binary, device, get_masks_gen.masks, nn_model_ref)
generator_data.create_data_g(table_of_truth)
#EVALUATE GOHR NN ON NEW DATASET
nn_model_ref.X_train_nn_binaire = generator_data.X_bin_train
nn_model_ref.X_val_nn_binaire = generator_data.X_bin_val
nn_model_ref.Y_train_nn_binaire = generator_data.Y_create_proba_train
nn_model_ref.Y_val_nn_binaire = generator_data.Y_create_proba_val
nn_model_ref.eval_all(["train", "val"])
net_linear = NN_linear(args)
net_linear.fc1.weight = nn_model_ref.net.fc1.weight.to(device)
net_linear.fc2.weight = nn_model_ref.net.fc2.weight.to(device)
net_linear.fc3.weight = nn_model_ref.net.fc3.weight.to(device)
net_linear.BN5.weight = nn_model_ref.net.BN5.weight.to(device)
net_linear.BN6.weight = nn_model_ref.net.BN6.weight.to(device)
net_all = nn_model_ref.net