def classifier_score(id_to_a_test, classifier, selector, X_test, y_test):
X_test = selector.transform(X_test)
y_test_predicted = classifier.predict(X_test)
tp = 0
tn = 0
fp = 0
fn = 0
for x in range(len(y_test)):
if y_test[x] == '1' and y_test_predicted[x] == '1':
tp += 1
if y_test[x] == '1' and y_test_predicted[x] == '0':
fn += 1
print("FN: " + id_to_a_test[x])
if y_test[x] == '0' and y_test_predicted[x] == '0':
tn += 1
if y_test[x] == '0' and y_test_predicted[x] == '1':
fp += 1
# print("FP: " + id_to_a_test[x])
print("Recall: " + str(tp / (tp + fn)))
print("Specificity: " + str(tn / (fp + tn)))
return {"TP": tp, "TN": tn, "FP": fp, "FN": fn,
"Balanced_Accuracy": balanced_accuracy_score(y_test, y_test_predicted),
"F_Measure": f1_score(y_test, y_test_predicted, pos_label='1'),
"Recall": recall_score(y_test, y_test_predicted, pos_label='1'),
"Negative-Recall": tn / (fp + tn),
"Precision": precision_score(y_test, y_test_predicted, pos_label='1'),
"Accuracy": classifier.score(X_test, y_test)}
def build_models_and_plot(self, X, y, model_dict, path, isplot, model_type, scoring_metric):
''' test a bunch of models and print out a sorted list of CV accuracies
inputs:
x: training data features, numpy array or Pandas dataframe
y: training data labels, numpy array or Pandas dataframe
model_dict: a dictionary of the form {name : model()}, where 'name' is a string
and 'model()' is a sci-kit-learn model object.
'''
n_folds=5
random_state = np.random.RandomState(0)
mean_aucs ={}
mean_f1 = {}
# Run classifier with cross-validation and plot ROC curves
for (name, model) in model_dict.items():
mean_aucs_list = []
tprs = []
aucs = []
print("Model: ", name)
cv = StratifiedKFold(n_splits=n_folds)
i = 0
scores = model_selection.cross_val_score(model, X, y, cv=n_folds, n_jobs=-1, scoring=scoring_metric)
print("Scores: ", scores)
plt.figure(figsize=(6, 4), dpi=150, facecolor='w', edgecolor='k')
for train, test in cv.split(X, y):
probas_ = model.fit(X[train], y[train] ).predict_proba(X[test])
y_pred = model.fit(X[train], y[train] ).predict(X[test])
if(scoring_metric=="balanced_accuracy"):
sm = balanced_accuracy_score(y[test], y_pred)
else:
sm = f1_score(y[test], y_pred, average='macro')
mean_aucs_list.append(self.auc_plotting(train, test, i, probas_, n_folds,
name, X,y, path, model_type, tprs,aucs))
i += 1
#print(set(y[test]), set(y_pred))
self.save_model(name + "_" + model_type, path, model)
mean_aucs[name]= [np.max(mean_aucs_list)]
mean_f1 [name] = [np.mean(scores)]
return mean_aucs, mean_f1
multi_true = regression_label_list[0].astype(np.int32)
multi_true = np.squeeze(multi_true)
elif multi_predictions_list != []:
multi_pred = np.array(multi_predictions_list)
for i in range(multi_pred.shape[1]):
multi_pred[0, i] = np.argmax(np.bincount(multi_pred[:, i]))
multi_pred = multi_pred[0]
multi_pred = np.squeeze(multi_pred)
multi_true = multi_label_list[0].astype(np.int32)
multi_true = np.squeeze(multi_true)
# change the standard 5-class predictions into the standard binary predictions
binary_pred = (multi_pred >= 2).astype(np.int32)
binary_true = (multi_true >= 2).astype(np.int32)
# calculate the accuracy, balanced accuracy score, confusion matrix, precision, recall and F1 score of the binary classification
binary_accuracy = metrics.accuracy_score(binary_true, binary_pred)
binary_balanced_accuracy = metrics.balanced_accuracy_score(binary_true, binary_pred)
binary_confusion_matrix = metrics.confusion_matrix(binary_true, binary_pred)
precision = metrics.precision_score(binary_true, binary_pred)
recall = metrics.recall_score(binary_true, binary_pred)
f1_score = metrics.f1_score(binary_true, binary_pred)
# output the result of the binary classification
print('Binary-accuracy:\n{}'.format(binary_accuracy))
print('Balanced Binary-accuracy:\n{}'.format(binary_balanced_accuracy))
print('Binary-confusion matrix:\n{}'.format(binary_confusion_matrix))
print('Precision:\n{}'.format(precision))
print('Recall:\n{}'.format(recall))
print('F1 score:\n{}'.format(f1_score))
# calculate the accuracy, balanced accuracy score and confusion matrix of the 5-calss classification
multi_accuracy = metrics.accuracy_score(multi_true, multi_pred)
multi_balanced_accuracy = metrics.balanced_accuracy_score(multi_true, multi_pred)
multi_confusion_matrix = metrics.confusion_matrix(multi_true, multi_pred)
#### svm
# Best: 0.724737 using {'probability': True, 'kernel': 'linear', 'gamma': 0.0001, 'degree': 3, 'C': 0.1}
svc = svm.SVC(random_state=8)
C = [0.001, 0.1, 1, 10]
gamma = [0.0001, 0.001, 0.01, 0.1]
degree = [1, 2, 3, 4]
kernel = ['linear', 'rbf', 'poly']
probability = [True]
random_grid = {'C': C, 'kernel': kernel, 'gamma': gamma, 'degree': degree, 'probability': probability}
random_search = RandomizedSearchCV(estimator=svc, param_distributions=random_grid, n_iter=50, scoring='accuracy', cv=3,
verbose=1, random_state=8)
random_search.fit(x_train, y_train)
print("Best: %f using %s" % (random_search.best_score_, random_search.best_params_))
svc = svm.SVC(random_state=8)
svc.fit(x_data_train, y_data_train)
svc_pred = svc.predict(x_test)
print(balanced_accuracy_score(y_test, svc_pred))
print(precision_score(y_test, svc_pred, average='weighted'))
print(recall_score(y_test, svc_pred, average='weighted'))
print(f1_score(y_test, svc_pred, average='weighted'))
import torch
from sklearn.metrics import accuracy_score, balanced_accuracy_score
from imblearn.datasets import make_imbalance
import numpy as np
from clstm import cnn,cnn_THS,X,Y
import sklearn
from sklearn.model_selection import train_test_split
vocab_size=100
#input_dim = 75
embedding_dim = 100
hidden_dim = 256
output_dim = 32
n_layers = 2
bidirectional = True
DROPOUT = 0.5
class_num=2
batch_size=32
dropout=0.5
model = cnn(vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, bidirectional,class_num,batch_size,output_dim, dropout)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=42, shuffle=True)
for i in range(len(X_train)):
model.partial_fit(np.asarray([X_train[i, :]]), np.asarray([y_train[i]]))
if i % 1000 == 0:
predictions = model.predict(X_test)
print("Online Accuracy: {}".format(balanced_accuracy_score(y_test, predictions)))
def test(args, io):
#test_loader = DataLoader(ModelNet40(partition='test', num_points=args.num_points),
# batch_size=args.test_batch_size, shuffle=True, drop_last=False)
test_loader = DataLoader(ModelNet40(partition='test', num_points=args.num_points),
batch_size=args.test_batch_size, shuffle=False, drop_last=False)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
if args.model == 'pointnet':
model = PointNet(args).to(device)
elif args.model == 'dgcnn':
model = DGCNN(args).to(device)
elif args.model == 'TransformerBaseline':
model = DGCNN_Transformer(args).to(device)
elif args.model == 'TemporalTransformer':
model = DGCNN_TemporalTransformer(args).to(device)
elif args.model == 'TemporalTransformer_v2':
model = DGCNN_TemporalTransformer_v2(args).to(device)
elif args.model == 'TemporalTransformer_v3':
model = DGCNN_TemporalTransformer_v3(args).to(device)
elif args.model == 'pi':
model = pi_DGCNN(args).to(device)
elif args.model == 'pi2':
model = pi_DGCNN_v2(args).to(device)
elif args.model == 'pipoint':
model = pipoint_DGCNN(args).to(device)
else:
raise Exception("Not implemented")
print(model)
print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))
model = nn.DataParallel(model)
if args.model_path:
if os.path.isfile(args.model_path):
print("=> loading checkpoint '{}'".format(args.model_path))
checkpoint = torch.load(args.model_path)
print(checkpoint)
if 'epoch' in checkpoint:
args.start_epoch = checkpoint['epoch']
#best_prec1 = checkpoint['best_prec1']
#best_prec5 = checkpoint['best_prec5']
if 'state_dict' in checkpoint:
model.load_state_dict(checkpoint['state_dict'], strict=False)
else:
model.load_state_dict(checkpoint, strict=False)
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.model_path, args.start_epoch))
else:
print("=> no checkpoint found at '{}'".format(args.model_path))
#model.load_state_dict(torch.load(args.model_path))
model = model.eval()
test_acc = 0.0
count = 0.0
test_true = []
test_pred = []
batch_time = AverageMeter()
end = time.time()
for i, (data, label) in enumerate(test_loader):
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
if args.model in ["pi", "pipoint", "pi2"]:
logits, atts = model(data)
elif args.model in ["TransformerBaseline", "TemporalTransformer", "TemporalTransformer_v2"]:
logits = model(data)
else:
logits, degree = model(data)
preds = logits.max(dim=1)[1]
test_true.append(label.cpu().numpy())
test_pred.append(preds.detach().cpu().numpy())
batch_time.update(time.time() - end)
end = time.time()
if i % 10 == 0:
print('Test {}, Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(i, batch_time=batch_time))
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
test_acc = metrics.accuracy_score(test_true, test_pred)
avg_per_class_acc = metrics.balanced_accuracy_score(test_true, test_pred)
per_class_acc = metrics.precision_score(test_true, test_pred, average=None)
outstr = 'Test :: test acc: %.6f, test avg acc: %.6f'%(test_acc, avg_per_class_acc)
outstr_2 = 'Test per class acc: {}'%per_class_acc
io.cprint(outstr)
io.cprint(outstr_2)
if args.model in ["pi", "pipoint", "pi2"]:
for j in range(4):
io.cprint('Att {} : {}'.format(j, atts[j].mean().item()))
pred_file = "D:\\Repos\\SJUMalwareEnsembleResearch\\Models\\Stacking\\Classic\\all_predictions5.sav"
pred_file2 = "D:\\Repos\\SJUMalwareEnsembleResearch\\Models\\Stacking\\Classic\\y_pred5.sav"
Y_pred = pickle.load(open(pred_file2, 'rb'))
pickle.dump(all_predictions, open(pred_file, 'wb'))
pickle.dump(Y_pred, open(pred_file2, 'wb'))
#Metrics
#Overall Accuracy
from sklearn.metrics import accuracy_score
accuracy_score(Y_test, Y_pred)
#Balanced Accuracy
from sklearn.metrics import balanced_accuracy_score
balanced_accuracy_score(Y_test, Y_pred)
#Precision, Recall, F1Score
from sklearn.metrics import precision_recall_fscore_support
precision_recall_fscore_support(Y_test, Y_pred, average='weighted')
from collections import Counter
from sklearn import metrics
mapping = Counter(Y_test)
#print(Counter(y_test))
mapping = dict(sorted(mapping.items()))
#--- 259.12324500083923 seconds ---
label_map = {
"0": "ADLOAD",
"1": "AGENT",
parameters = open("parameters.yml")
yamlparameters = yaml.load(parameters,Loader=yaml.FullLoader)
opt = Optimizer(config, api_key=yamlparameters["comet_api_key"], project_name="newhitmaskGBDTclassifier",auto_metric_logging=True)
X_train, X_test, y_train, y_test = get_features(yamlparameters["DataDir"])
for experiment in opt.get_experiments():
model = xgboost.XGBClassifier(booster="gbtree",verbosity=1,objective="binary:logistic",tree_method="exact",
learning_rate=experiment.get_parameter("learning_rate"),
gamma=experiment.get_parameter("gamma"),
n_estimators=experiment.get_parameter("n_estimators"),
max_depth=experiment.get_parameter("max_depth"),
min_child_weight=experiment.get_parameter("min_child_weight"),
subsample=experiment.get_parameter("subsample"),
alpha=experiment.get_parameter("alpha"),n_jobs=4)
model.fit(X_train,y_train)
y_predict = model.predict(X_test)
binary_accuracy = metrics.balanced_accuracy_score(y_test,y_predict)
y_predict_prob = model.predict_proba(X_test)[:,1]
auc = metrics.roc_auc_score(y_test,y_predict_prob)
print("AUC:",auc)
print("ACC:",binary_accuracy)
experiment.log_metric("ROC",auc)
experiment.log_metric("Binary_Accuracy",binary_accuracy)
random.choice(['M', 'S']) for i in range(len(y_test))
])
classifier_rf = RandomForestClassifier(n_estimators=5)
classifier_rf.fit(X_train, y_train)
puntos_predichos_rf = classifier_rf.predict(X_test)
classifier_svm = SVC()
classifier_svm.fit(valores_pixeles, clase)
puntos_predichos_svm = classifier_svm.predict(X_test)
# Observo metricas
# ==================================================================================
print("---------------------------------")
print("Accuracy_score")
print("---------------------------------")
print('RANDOM', accuracy_score(y_test, puntos_predichos_aleatorio))
print('RF:', accuracy_score(y_test, puntos_predichos_rf))
print('SVM:', accuracy_score(y_test, puntos_predichos_svm))
print("---------------------------------")
print("Balanced Accuracy_score")
print("---------------------------------")
print('RANDOM', balanced_accuracy_score(y_test, puntos_predichos_aleatorio))
print('RF:', balanced_accuracy_score(y_test, puntos_predichos_rf))
print('SVM:', balanced_accuracy_score(y_test, puntos_predichos_svm))
plot_confusion_matrix(classifier_svm, X_test, y_test)
plt.title("CM SVM")
plt.show()
# %% [markdown]
# With the dummy classifier, which always predicts the negative class `'not
# donated'`, we obtain an accuracy score of 76%. Therefore, it means that this
# classifier, without learning anything from the data `data`, is capable of
# predicting as accurately as our logistic regression model.
#
# The problem illustrated above is also known as the class imbalance problem.
# When the classes are imbalanced, accuracy should not be used. In this case,
# one should either use the precision and recall as presented above or the
# balanced accuracy score instead of accuracy.
# %%
from sklearn.metrics import balanced_accuracy_score
balanced_accuracy = balanced_accuracy_score(target_test, target_predicted)
print(f"Balanced accuracy: {balanced_accuracy:.3f}")
# %% [markdown]
# The balanced accuracy is equivalent to accuracy in the context of balanced
# classes. It is defined as the average recall obtained on each class.
#
# ## Evaluation and different probability thresholds
#
# All statistics that we presented up to now rely on `classifier.predict` which
# outputs the most likely label. We haven't made use of the probability
# associated with this prediction, which gives the confidence of the
# classifier in this prediction. By default, the prediction of a classifier
# corresponds to a threshold of 0.5 probability in a binary classification
# problem. We can quickly check this relationship with the classifier that
# we trained.
loss_train = loss(output[train_idx], y[train_idx])
loss_train.backward()
optimizer.step()
model.eval()
with torch.no_grad():
output = model(x, A, gene_A)
loss_test = loss(output[test_idx], y[test_idx])
print(
'Epoch: {:04d}'.format(i),
'train_loss: {:.4f}'.format(loss_train.item()),
'train_roc_auc: {:.4f}'.format(
metrics.roc_auc_score(yy[train_idx].cpu(),
output[train_idx].cpu())),
'train_balance_acc_train: {:.4f}'.format(
metrics.balanced_accuracy_score(
y[train_idx].cpu(),
output[train_idx].argmax(dim=1).cpu())),
'test_roc_auc: {:.4f}'.format(
metrics.roc_auc_score(yy[test_idx].cpu(),
output[test_idx].cpu())),
'test_balance_acc_test: {:.4f}'.format(
metrics.balanced_accuracy_score(
y[test_idx].cpu(),
output[test_idx].argmax(dim=1).cpu())))
# loss_train_all.append(loss_train.item())
# train_bacc_all.append(metrics.balanced_accuracy_score(y[train_idx].cpu(), output[train_idx].argmax(dim=1).cpu()))
# loss_test_all.append(loss_test.item())
# test_bacc_all.append(metrics.balanced_accuracy_score(y[test_idx].cpu(), output[test_idx].argmax(dim=1).cpu()))
# plot_train_test_loss(loss_train_all, train_bacc_all, loss_test_all, test_bacc_all)
# print('confusion_matrix: ', confusion_matrix(y[test_idx].cpu(), output[test_idx].argmax(dim=1).cpu()))
def train(args, io):
if args.dataset == 'modelnet40':
train_loader = DataLoader(ModelNet40(partition='train',
num_points=args.num_points),
num_workers=8,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
test_loader = DataLoader(ModelNet40(partition='test',
num_points=args.num_points),
num_workers=8,
batch_size=args.test_batch_size,
shuffle=True,
drop_last=False)
else:
pcd = np.empty((57449, 2048, 3), dtype=np.float32)
label = np.empty((57449, ), dtype=np.int64)
i = 0
j = 0
for fd in glob.glob('./data/shapenet/0*'):
for f in os.listdir(fd):
pct = o3d.io.read_point_cloud(os.path.join(fd, f))
pcd[i] = np.asarray(pct.points)
label[i] = j
i = i + 1
j = j + 1
if i != 57449 or j != 57:
raise ValueError('data stat doesn\'t match')
pcd_train, pcd_test, label_train, label_test = train_test_split(
pcd, label, stratify=label, random_state=2215)
train_loader = DataLoader(shapeNetTrain(p=pcd_train,
label=label_train,
num_points=args.num_points),
num_workers=8,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
test_loader = DataLoader(shapeNetTest(p=pcd_test,
label=label_test,
num_points=args.num_points),
num_workers=8,
batch_size=args.test_batch_size,
shuffle=True,
drop_last=False)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
if args.model == 'GSNET':
if args.dataset == 'shapenet':
model = GSNET(args, output_channels=57).to(device)
else:
model = GSNET(args).to(device)
elif args.model == 'NewGSNET':
print('NewGSNET!')
if args.dataset == 'shapenet':
model = NewGSNET(args, output_channels=57).to(device)
else:
model = NewGSNET(args).to(device)
else:
raise Exception("Not implemented")
print(str(model))
print(sum(p.numel() for p in model.parameters() if p.requires_grad))
logfile.write(str(model))
model = nn.DataParallel(model)
print("Let's use", torch.cuda.device_count(), "GPUs!")
if args.use_sgd:
print("Use SGD")
logfile.write("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
print("Use Adam")
logfile.write("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=0.00001)
criterion = cal_loss
best_test_acc = 0
for epoch in range(args.epochs):
scheduler.step()
####################
# Train
####################
train_loss = 0.0
count = 0.0
model.train()
train_pred = []
train_true = []
for data, label in train_loader:
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
opt.zero_grad()
logits = model(data)
loss = criterion(logits, label)
loss.backward()
opt.step()
preds = logits.max(dim=1)[1]
count += batch_size
train_loss += loss.item() * batch_size
train_true.append(label.cpu().numpy())
train_pred.append(preds.detach().cpu().numpy())
train_true = np.concatenate(train_true)
train_pred = np.concatenate(train_pred)
outstr = 'Train %d, loss: %.6f, train acc: %.6f, train avg acc: %.6f' % (
epoch, train_loss * 1.0 / count,
metrics.accuracy_score(train_true, train_pred),
metrics.balanced_accuracy_score(train_true, train_pred))
io.cprint(outstr)
logfile.write(outstr)
####################
# Test
####################
test_loss = 0.0
count = 0.0
model.eval()
test_pred = []
test_true = []
for data, label in test_loader:
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
logits = model(data)
loss = criterion(logits, label)
preds = logits.max(dim=1)[1]
count += batch_size
test_loss += loss.item() * batch_size
test_true.append(label.cpu().numpy())
test_pred.append(preds.detach().cpu().numpy())
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
test_acc = metrics.accuracy_score(test_true, test_pred)
avg_per_class_acc = metrics.balanced_accuracy_score(
test_true, test_pred)
outstr = 'Test %d, loss: %.6f, test acc: %.6f, test avg acc: %.6f' % (
epoch, test_loss * 1.0 / count, test_acc, avg_per_class_acc)
io.cprint(outstr)
logfile.write(outstr)
if test_acc >= best_test_acc:
best_test_acc = test_acc
io.cprint('Max Acc:%.6f' % best_test_acc)
logfile.write('\nMax Acc:%.6f' % best_test_acc)
torch.save(model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
def test(args, io):
if args.dataset == 'modelnet40':
test_loader = DataLoader(ModelNet40(partition='test',
num_points=args.num_points),
num_workers=8,
batch_size=args.test_batch_size,
shuffle=True,
drop_last=False)
else:
pcd = np.empty((57449, 2048, 3), dtype=np.float32)
label = np.empty((57449, ), dtype=np.int64)
i = 0
j = 0
for fd in glob.glob('./data/shapenet/0*'):
for f in os.listdir(fd):
pct = o3d.io.read_point_cloud(os.path.join(fd, f))
pcd[i] = np.asarray(pct.points)
label[i] = j
i = i + 1
j = j + 1
if i != 57449 or j != 57:
raise ValueError('data stat doesn\'t match')
pcd_train, pcd_test, label_train, label_test = train_test_split(
pcd, label, stratify=label, random_state=1152)
test_loader = DataLoader(shapeNetTest(p=pcd_test,
label=label_test,
num_points=args.num_points),
num_workers=8,
batch_size=args.test_batch_size,
shuffle=True,
drop_last=False)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
# if args.dataset == 'modelnet40':
# model = GSNET(args).to(device)
# else:
# model = GSNET(args,output_channels=57).to(device)
if args.model == 'GSNET':
if args.dataset == 'shapenet':
model = GSNET(args, output_channels=57).to(device)
else:
model = GSNET(args).to(device)
elif args.model == 'NewGSNET':
print('NewGSNET!')
if args.dataset == 'shapenet':
model = NewGSNET(args, output_channels=57).to(device)
else:
model = NewGSNET(args).to(device)
else:
raise Exception("Not implemented")
model = nn.DataParallel(model)
model.load_state_dict(torch.load(args.model_path))
model = model.eval()
test_acc = 0.0
count = 0.0
test_true = []
test_pred = []
cnt = 0
for data, label in test_loader:
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
logits = model(data)
preds = logits.max(dim=1)[1]
if cnt <= 25:
for index, inq in enumerate(
preds.detach().cpu().numpy() != label.cpu().numpy()):
if inq:
pcd = o3d.geometry.PointCloud()
# print(data.cpu().permute(0,2,1).numpy()[index].shape)
pcd.points = o3d.utility.Vector3dVector(
(data.cpu().permute(0, 2, 1).numpy())[index])
o3d.io.write_point_cloud(
"./errors/gsnet/{:d}-{:s}-{:s}.ply".format(
cnt, folderTclass[(
preds.detach().cpu().numpy())[index]],
folderTclass[(label.cpu().numpy())[index]]), pcd)
cnt = cnt + 1
test_true.append(label.cpu().numpy())
test_pred.append(preds.detach().cpu().numpy())
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
print(np.sum(test_true != test_pred))
test_acc = metrics.accuracy_score(test_true, test_pred)
avg_per_class_acc = metrics.balanced_accuracy_score(test_true, test_pred)
outstr = 'Test :: test acc: %.6f, test avg acc: %.6f' % (test_acc,
avg_per_class_acc)
io.cprint(outstr)
logfile.write(outstr)
def evaluate_model(est, x_tr, y_tr, x_ho, y_ho, **kwargs):
pred_tr = est.predict(x_tr)
score_tr = balanced_accuracy_score(y_tr, pred_tr)
pred_ho = est.predict(x_ho)
score_ho = balanced_accuracy_score(y_ho, pred_ho)
return {'score-tr': score_tr, 'score-ho': score_ho}
def log_model_metrics(y_actual: np.ndarray, y_predicted: np.ndarray) -> None:
"""log.info model evaluation metrics to stdout."""
accuracy = balanced_accuracy_score(y_actual, y_predicted, adjusted=True)
f1 = f1_score(y_actual, y_predicted, average="weighted")
mlflow.log_metric("accuracy", accuracy)
mlflow.log_metric("f1", f1)
random_state=22,
kernel='linear',
C=best_params[dict_dataset[dataset] + 'SVM.' +
str(i + 1) + '.' + str(j + 1)]['C'],
probability=True)
np.random.set_state(state)
gkf = GroupKFold(n_splits=5).split(X_train, Y[train_idx],
cluster_in[train_idx])
for k, (train, test) in enumerate(gkf):
clf.fit(X_train[train], Y[train_idx][train])
acc.append(
accuracy_score(Y[train_idx][test],
clf.predict(X_train[test])))
bal_acc.append(
balanced_accuracy_score(Y[train_idx][test],
clf.predict(X_train[test])))
rec.append(
recall_score(Y[train_idx][test],
clf.predict(X_train[test]),
pos_label=1))
pre.append(
precision_score(Y[train_idx][test],
clf.predict(X_train[test]),
pos_label=1))
rocauc.append(
roc_auc_score(Y[train_idx][test],
clf.predict_proba(X_train[test])[:, 1]))
precision, recall, _ = precision_recall_curve(
Y[train_idx][test],
clf.predict_proba(X_train[test])[:, 1])
aupr.append(auc(recall, precision))
C=C,
seed=2 * 722019 + seed,
fn=fn,
device=device
) # 2*722019+seed is just to have a large odd number for seeding that is recommended for generating random numbers, fixed for reproducibility
[X_tr_nmf, X_val_nmf, X_te_nmf, H] = m.fit_transform()
chkpt = torch.load(fn)
m.load_state_dict(chkpt['state_dict'])
# best_iter = chkpt['epoch']
accval = chkpt['best_val_acc']
m.eval()
y_tr_pred = m.predict(X_tr_nmf, pts_tr)
y_te_pred = m.predict(X_te_nmf, pts_te)
acctr = balanced_accuracy_score(y_train, y_tr_pred)
accte = accuracy_score(y_test, y_te_pred)
balaccte = balanced_accuracy_score(y_test, y_te_pred)
preciste = precision_score(y_test, y_te_pred)
recallte = recall_score(y_test, y_te_pred)
f1te = f1_score(y_test, y_te_pred)
w2 = np.square(
m.state_dict()['fc.weight'].cpu().numpy()).sum(axis=None)
b2 = np.square(
m.state_dict()['fc.bias'].cpu().numpy()).sum(axis=None)
w = 1 / pd.Series(y_train).value_counts(
normalize=True).sort_index().to_numpy()
vce = chkpt['celoss']
# err = np.vstack((X_train, X_val, X_test)) - np.vstack((X_tr_nmf, X_val_nmf, X_te_nmf)) @ H
def run(prefix, Kmax, restart, trial, modeltype='regression', rftype='R', treenum=100, depth=20, maxitr=1000, tol=1e-6, njobs=1,smear_num=100,verbose=True,scoring='standard'):
# trial
dirname = './result/result_%s_itr' % (prefix,)
for t in range(trial):
if verbose:
print('\n***********************************')
print('Fold: %02d/%02d' %(t+1,trial))
print( '***********************************')
# data
dirname2 = '%s/result_%02d' % (dirname, t)
trfile = '%s/%s_train_%02d.csv' % (dirname2, prefix, t)
tefile = '%s/%s_test_%02d.csv' % (dirname2, prefix, t)
Ztr = pd.read_csv(trfile, delimiter=',', header=None).values
Xtr = Ztr[:, :-1]
ytr = Ztr[:, -1]
Zte = pd.read_csv(tefile, delimiter=',', header=None).values
Xte = Zte[:, :-1]
yte = Zte[:, -1]
if (rftype=='R'):
# build R random forest
if modeltype == 'regression':
os.system('Rscript ./baselines/buildRegForest.R %s %s %s %d 0' % (trfile, tefile, dirname2, treenum))
elif modeltype == 'classification':
if verbose: print('Running buildClfForest.R')
os.system('Rscript ./baselines/buildClfForest.R %s %s %s %d 0' % (trfile, tefile, dirname2, treenum))
if verbose: print('Running DefragModel.parseRtrees')
splitter = DefragModel.parseRtrees('%s/forest/' % (dirname2,))
zfile = '%s/pred_test.csv' % (dirname2,)
zte = pd.read_csv(zfile, delimiter=' ', header=None).values[:, -1]
if modeltype == 'regression':
score = np.mean((yte - zte)**2)
cover = 1.0
coll = 1.0
elif modeltype == 'classification':
if (scoring=='balanced'):
from sklearn.metrics import balanced_accuracy_score
score = 1.0 - balanced_accuracy_score(yte,zte)
elif (scoring=='standard'):
from sklearn.metrics import accuracy_score
score = 1.0 - accuracy_score(yte,zte)
elif (scoring=='f1'):
from sklearn.metrics import f1_score
score = f1_score(yte,zte)
cover = 1.0
coll = 1.0
elif (rftype=='SL'):
if modeltype=='regression':
forest = RandomForestRegressor(n_estimators=treenum,n_jobs=njobs)
forest.fit(Xtr,ytr)
score = mean_squared_error(yte,forest.predict(Xte))
elif modeltype == 'classification':
forest = RandomForestClassifier(n_estimators=treenum,n_jobs=njobs, max_depth=depth)
forest.fit(Xtr,ytr)
# parameters = {'max_leaf_nodes':[16, 32, 64, 128], 'min_samples_leaf':[10,50,100,150]}
# forest = RandomForestClassifier(n_estimators=treenum,n_jobs=njobs)
# gs = GridSearchCV(forest, parameters, cv=3)
# gs.fit(Xtr,ytr)
# print(gs.best_params_)
# forest = gs.best_estimator_
if (scoring=='balanced'):
from sklearn.metrics import balanced_accuracy_score
score = 1.0 - balanced_accuracy_score(yte,zte)
elif (scoring=='standard'):
from sklearn.metrics import accuracy_score
score = 1.0 - accuracy_score(yte,zte)
elif (scoring=='f1'):
from sklearn.metrics import f1_score
score = f1_score(yte,zte)
cover = 1.0
coll = 1.0
# parse sklearn tree ensembles into the array of (feature index, threshold)
splitter = DefragModel.parseSLtrees(forest)
# Write sklearn random forests to file (in the same format as the .R ones)
# inTrees, NHarvest, BATrees and DTree2 can then be performed on this RF without any modification to packages.
DefragModel.save_SLtrees('%s/forest/' % (dirname2,),forest)
print('RF Test Score = %.2f' % (score))
print('RF Test Coverage = %.2f' % (cover))
print('RF Overlap = %.2f' % (coll))
np.savetxt('%s/res_rf_%02d.csv' % (dirname2, t), np.array([score, cover, coll, -1]), delimiter=',')
# defragTrees
if verbose: print('Defragging model')
mdl = DefragModel(modeltype=modeltype, restart=restart, maxitr=maxitr, tol=tol, seed=restart*t, njobs=njobs,score=scoring)
if verbose: print('Fitting defrag')
mdl.fit(Xtr, ytr, splitter, Kmax, fittype='FAB')
joblib.dump(mdl, '%s/%s_defrag_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl.evaluate(Xte, yte)
print('Defrag Test Score = %.2f' % (score))
print('Defrag Test Coverage = %.2f' % (cover))
print('Defrag Overlap = %.2f' % (coll))
np.savetxt('%s/res_defrag_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl.rule_)]), delimiter=',')
if (rftype=='R'):
# inTrees
if verbose: print('Fitting inTree model')
mdl2 = inTreeModel(modeltype=modeltype,score=scoring)
mdl2.fit(Xtr, ytr, '%s/inTrees.txt' % (dirname2,))
joblib.dump(mdl2, '%s/%s_inTrees_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl2.evaluate(Xte, yte)
print('inTrees Test Score = %.2f' % (score))
print('inTrees Test Coverage = %.2f' % (cover))
print('inTrees Overlap = %.2f' % (coll))
np.savetxt('%s/res_inTrees_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl2.rule_)]), delimiter=',')
# NHarvest
if verbose: print('Fitting NHarvest model')
mdl3 = NHarvestModel(modeltype=modeltype,score=scoring)
mdl3.fit(Xtr, ytr, '%s/nodeHarvest.txt' % (dirname2,))
joblib.dump(mdl3, '%s/%s_nodeHarvest_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl3.evaluate(Xte, yte)
zfile = '%s/pred_test_nh.csv' % (dirname2,)
zte = pd.read_csv(zfile, delimiter=' ', header=None).values[:, -1]
if modeltype == 'regression':
score = np.mean((yte - zte)**2)
elif modeltype == 'classification':
if (score=='balanced'):
from sklearn.metrics import balanced_accuracy_score
score = 1.0 - balanced_accuracy_score(yte,zte)
elif (score=='standard'):
from sklearn.metrics import accuracy_score
score = 1.0 - accuracy_score(yte,zte)
elif (score=='f1'):
from sklearn.metrics import f1_score
score = f1_score(yte,zte)
print('NHarvest Test Score = %.2f' % (score))
print('NHarvest Test Coverage = %.2f' % (cover))
print('NHarvest Overlap = %.2f' % (coll))
np.savetxt('%s/res_nodeHarvest_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl3.rule_)]), delimiter=',')
# BA Tree Result
if verbose: print('Fitting BATree model')
mdl4 = BTreeModel(modeltype=modeltype, njobs=njobs, seed=t, smear_num=50,score=scoring)
mdl4.fit(Xtr, ytr, '%s/forest/' % (dirname2,))
joblib.dump(mdl4, '%s/%s_BATree_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl4.evaluate(Xte, yte)
print('BATree Test Score = %.2f' % (score))
print('BATree Test Coverage = %.2f' % (cover))
print('BATree Overlap = %.2f' % (coll))
np.savetxt('%s/res_BATree_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl4.rule_)]), delimiter=',')
BATree_depth = mdl4.tree.max_depth_
# DTree - depth = 2
if verbose: print('Fitting Dtree - depth=2')
mdl5 = DTreeModel(modeltype=modeltype, max_depth=[2],score=scoring)
mdl5.fit(Xtr, ytr)
joblib.dump(mdl5, '%s/%s_DTree2_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl5.evaluate(Xte, yte)
print('DTree2 Test Score = %.2f' % (score))
print('DTree2 Test Coverage = %.2f' % (cover))
print('DTree2 Overlap = %.2f' % (coll))
np.savetxt('%s/res_DTree2_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl5.rule_)]), delimiter=',')
# DTree - depth = BATree_depth
if verbose: print('Fitting Dtree to match BATree depth = ', BATree_depth)
mdl5 = DTreeModel(modeltype=modeltype, max_depth=[BATree_depth],score=scoring)
mdl5.fit(Xtr, ytr)
joblib.dump(mdl5, '%s/%s_DTreeBA_%02d.mdl' % (dirname2, prefix, t), compress=9)
score, cover, coll = mdl5.evaluate(Xte, yte)
print('DTreeBA Test Score = %.2f' % (score))
print('DTreeBA Test Coverage = %.2f' % (cover))
print('DTreeBA Overlap = %.2f' % (coll))
np.savetxt('%s/res_DTreeBA_%02d.csv' % (dirname2, t), np.array([score, cover, coll, len(mdl5.rule_)]), delimiter=',')
# summary
plot_summarize(prefix, trial, rftype)
summary2csv(prefix, trial, rftype)
def SM_test(summaryMeasure,
k,
data,
label,
data_test,
label_test,
batch_size,
is_mmens=False):
train_split_k = train_split[k]
val_split_k = val_split[k]
#reset the model before new cv
model = CNN_model()
model.to(device)
# Loss and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(),
momentum=0.9,
lr=0.001,
weight_decay=1e-3)
# load data
train_split_k = train_split[k]
testData = load_data(
split='test',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label, # need train data for mean and sd normalization
data_test=data_test,
label_test=label_test)
test_loader = DataLoader(dataset=testData,
batch_size=batch_size,
shuffle=True)
# set and load best model
model = CNN_model()
model.load_state_dict(
torch.load(MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
'_CV' + str(k + 1) + '.pt'))
model.to(device)
# START TRAINING FOR THIS CV
# TEST EVALUATION
model.eval()
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
current_ep_acc = 0.0
current_ep_loss = 0.0
total_step_v = len(test_loader)
# F1 calc
F1_labels = []
F1_pred = []
for i, (images, labels
) in enumerate(test_loader): #TEST THE MODEL ON THE VALIDATION
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels), 1).type(torch.cuda.FloatTensor)
outputs_v = model(images) # the forward uses the entire batch together
predicted = outputs_v.data > 0.0 #define True if >0.5 and False if <0.5
# collect true labels and predicted for metrix calc
if i == 0:
F1_labels = labels.int().cpu().numpy()
F1_pred = predicted.int().cpu().numpy()
else:
F1_labels = np.concatenate((F1_labels, labels.int().cpu().numpy()))
F1_pred = np.concatenate((F1_pred, predicted.int().cpu().numpy()))
loss_v = criterion(outputs_v, labels)
total_v = labels.size(
0) # get it in case the last batch has different number of sample
correct_v = (predicted == labels
).sum().item() # sum the correct answers for this batch
#acc_list_v.append(correct_v / total_v) # n of correct / n of sample in this batch
total_ep += total_v # sum all the samples. it must end up being the tot training set
correct_ep += correct_v # sum together the right answers in entire training set
loss_ep += (loss_v.item() * len(labels))
current_ep_acc = (correct_ep / total_ep) * 100
current_ep_loss = loss_ep / total_ep
acc = accuracy_score(F1_labels, F1_pred)
bal_acc = balanced_accuracy_score(F1_labels, F1_pred)
F1_Score = f1_score(F1_labels, F1_pred,
average='weighted') #, zero_division=1)
tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()
print('CV {}, sumMeasure: {} Test Accuracy of the model is: {}, F1: {}%'.
format(k + 1, summaryMeasure, current_ep_acc, F1_Score))
if is_mmens == True:
return F1_labels.reshape(-1), F1_pred.reshape(-1)
else:
return bal_acc, f1_score
)
print(
f"\nBondad del modelo de Regresión Logística con características estandarizadas para el modelo {nombre}"
)
# Para el cálculo de errores, hacemos una predicción sobre el conjunto Test separado inicialmente
# y calculamos la precisión balanceada. Para expresarlo en términos del error, hacemos 1 - accuracy
y_pred = clf_rl.predict(X_train)
print(f"Ein = {1-balanced_accuracy_score(y_train[:, i], y_pred)}")
y_pred = clf_rl.predict(X_test)
print(f"Etest = {1-balanced_accuracy_score(y_test[:, i], y_pred)}")
print(f"Ecv = {1-clf_rl.best_score_}")
# Almacenamos este modelo como el mejor estimador, de cara a compararlo con los otros a utilizar
best_estimator = clf_rl
best_name = "Regresión Logística"
best_score = balanced_accuracy_score(y_test[:, i], y_pred)
stop()
# Apoyándonos en la función de Scikit-Learn, graficamos la matriz de confusión resultante para el conjunto Test.
plot_confusion_matrix(clf_rl.best_estimator_,
X_test,
y_test[:, i],
display_labels=labels[i],
values_format='d')
plt.title(
f"Matriz de confusión para el caso de {nombre}\n usando Regresión Logística"
)
plt.ylabel(f"Clase verdadera")
plt.xlabel(f"Clase predicha")
plt.show()
def SM_train_val_kfoldCV(summaryMeasure, data, label, batch_size, num_epochs,
n_split):
# CROSSVALIDATION LOOP
for k in np.arange(n_split):
train_split_k = train_split[k]
val_split_k = val_split[k]
#reset the model before new cv
model = CNN_model()
model.to(device)
# Loss and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(),
momentum=0.9,
lr=0.001,
weight_decay=1e-3)
# load data
trainData = load_data(split='train',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label)
valData = load_data(split='val',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label)
train_loader = DataLoader(dataset=trainData,
batch_size=batch_size,
shuffle=True)
val_loader = DataLoader(dataset=valData,
batch_size=batch_size,
shuffle=True)
# START TRAINING FOR THIS CV
# collect loss and acc of each epoch
epoch_loss_train = []
epoch_acc_train = []
epoch_loss_val = []
epoch_balacc_val = []
epoch_F1_Score = []
best_acc = None
# TRAIN THE MODEL
for epoch in range(num_epochs):
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
model.train()
for i, (images, labels) in enumerate(train_loader):
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels),
1).type(torch.cuda.FloatTensor)
# Run the forward pass
outputs = model(images)
loss = criterion(outputs, labels)
# Backprop and perform optimisation
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Track the accuracy
total = labels.size(0)
predicted = outputs.data > 0.0
correct = (predicted == labels).sum().item()
total_ep += total
correct_ep += correct
loss_ep += (loss.item() * len(labels))
epoch_acc_train.append((correct_ep / total_ep))
epoch_loss_train.append(loss_ep / total_ep)
# Test the model on validation data
model.eval()
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
# F1 calc
F1_labels = []
F1_pred = []
for i, (images, labels) in enumerate(val_loader):
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels),
1).type(torch.cuda.FloatTensor)
outputs_v = model(
images) # the forward uses the entire batch together
predicted = outputs_v.data > 0.0 #define True if >0.5 and False if <0.5
# collect true labels and predicted for metrix calc
if i == 0:
F1_labels = labels.int().cpu().numpy()
F1_pred = predicted.int().cpu().numpy()
else:
F1_labels = np.concatenate(
(F1_labels, labels.int().cpu().numpy()))
F1_pred = np.concatenate(
(F1_pred, predicted.int().cpu().numpy()))
loss_v = criterion(outputs_v, labels)
total_v = labels.size(0)
correct_v = (predicted == labels).sum().item()
total_ep += total_v
correct_ep += correct_v
loss_ep += (loss_v.item() * len(labels))
# calculate metrices of this epoch
current_ep_loss = loss_ep / total_ep
current_ep_acc = (correct_ep / total_ep)
acc = accuracy_score(F1_labels, F1_pred)
balacc = balanced_accuracy_score(F1_labels, F1_pred)
F1_Score = f1_score(F1_labels, F1_pred, average='weighted')
tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()
if not best_acc or best_acc < current_ep_acc:
best_acc = current_ep_acc
torch.save(
model.state_dict(),
MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
'_CV' + str(k + 1) + '.pt')
# collect matrixes of all the epochs
epoch_loss_val.append((loss_ep / total_ep))
epoch_balacc_val.append(balacc)
print('{}, CV_{}, best accuracy = {}'.format(summaryMeasure, k + 1,
acc))
def eval_whitebox_classifier(R,
g,
EX,
StdX,
NormV,
x0,
label_x0,
bb_classifier,
wb_name,
precision_recalls=False):
# scale x0 in the ridge model space
sx0 = np.divide((x0 - EX), StdX, where=np.logical_not(np.isclose(StdX, 0)))
# sx0 = np.divide((x0 - EX), StdX, where=StdX!=0)
# compute the p-score of sx0
sx0_w = np.dot(sx0, g.coef_)
p_score = sx0_w + g.intercept_
if linalg.norm(g.coef_) < 1.0e-5 or (
abs(sx0_w) < 1.0e-5
): # or math.isclose(p_score, 0) or math.isclose(p_score, 1):
N_sx0_w = np.zeros(len(x0))
R.wb_plane_dist_x0 = 0.0
else:
N_sx0_w = np.multiply(sx0, (0.5 - p_score) / sx0_w)
R.wb_plane_dist_x0 = p_score / linalg.norm(g.coef_)
# get the boundary point x1
sx1 = sx0 + N_sx0_w
x1 = (sx1 * StdX) + EX
prob_x1 = bb_classifier([x1])[0]
R.wb_class_x1 = 1 if prob_x1[1] > prob_x1[0] else 0
R.wb_prob_x1_F = prob_x1[0]
R.wb_prob_x1_T = prob_x1[1]
R.wb_prob_x1_c0 = prob_x1[label_x0]
R.wb_local_discr = g.predict([sx0])[0] - R.prob_x0
R.wb_boundary_discr = g.predict([sx1])[0] - prob_x1[0]
# build the (scaled) neighborhood of x0
SNX0 = np.tile(sx0,
(NormV.shape[0], 1)) # repeat T times the scaled x1 row
SNX0 = SNX0 + NormV
NX0 = (SNX0 * StdX) + EX
# build the (scaled) neighborhood of x1
SNX1 = np.tile(sx1,
(NormV.shape[0], 1)) # repeat T times the scaled x1 row
SNX1 = SNX1 + NormV
NX1 = (SNX1 * StdX) + EX
# predict the instance classes using the Black-Box and the White-Box classifiers
BBY0, WBY0 = bb_classifier(NX0)[:, 0], g.predict(SNX0)
BBY1, WBY1 = bb_classifier(NX1)[:, 0], g.predict(SNX1)
if label_x0 == 1:
WBY0, WBY1 = 1 - WBY0, 1 - WBY1
BBCLS0, WBCLS0 = BBY0 > 0.5, WBY0 > 0.5
BBCLS1, WBCLS1 = BBY1 > 0.5, WBY1 > 0.5
R.wb_x1_change_score = np.mean(BBCLS1 != label_x0)
R.wb_avg_bb_nx0 = np.mean(BBY0)
R.wb_avg_bb_nx1 = np.mean(BBY1)
R.wb_ratio_x0 = np.mean(BBCLS0)
R.wb_ratio_x1 = np.mean(BBCLS1)
R.wb_ratio_wb_x0 = np.mean(WBCLS0)
R.wb_ratio_wb_x1 = np.mean(WBCLS1)
try:
R.wb_fidelity = accuracy_score(BBCLS0, WBCLS0)
R.wb_prescriptivity = accuracy_score(BBCLS1, WBCLS1)
R.wb_bal_fidelity = balanced_accuracy_score(BBCLS0, WBCLS0)
R.wb_bal_prescriptivity = balanced_accuracy_score(BBCLS1, WBCLS1)
R.wb_fidelity_f1 = f1_score(BBCLS0, WBCLS0)
R.wb_prescriptivity_f1 = f1_score(BBCLS1, WBCLS1)
# print(sklearn.metrics.confusion_matrix(BBCLS1, WBCLS1), wb_name)
if precision_recalls:
R.wb_precision_x1 = precision_score(BBCLS1, WBCLS1)
R.wb_recall_x1 = recall_score(BBCLS1, WBCLS1)
# R.wb_fidelity_R2 = g.score(SNX0, BBY0)
# R.wb_prescriptivity_R2 = g.score(SNX1, BBY1)
except:
R.wb_bal_fidelity, R.wb_bal_prescriptivity = 0, 0
R.wb_fidelity, R.wb_prescriptivity = 0, 0
# R.wb_fidelity_R2, R.wb_prescriptivity_R2 = 0, 0
R.wb_fidelity_f1, R.wb_prescriptivity_f1 = 0, 0
# rename R keys (wb_* -> wb_name_*)
for key in copy.copy(list(R.__dict__.keys())):
if key.startswith("wb_"):
R.__dict__[wb_name + key[2:]] = R.__dict__.pop(key)
return (x1, sx1)
amsgrad=False)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
model.fit(x=X_train,
y=Y_train_trans,
batch_size=batch_size,
epochs=nr_epochs,
validation_data=(X_val, Y_val_trans),
callbacks=[early_stop, reduce_lr],
verbose=1)
Y_val_predicted = np.argmax(model.predict(X_val), axis=1)
nn_val_acc = accuracy_score(Y_val, Y_val_predicted)
nn_val_bal_acc = balanced_accuracy_score(Y_val, Y_val_predicted)
print('Neural network accuracy on val set: {0} \n'.format(
np.round(nn_val_acc, 2)))
print('Neural network bal acc on val set: {0} \n'.format(
np.round(nn_val_bal_acc, 2)))
Y_test_predicted = np.argmax(model.predict(X_test), axis=1)
nn_test_acc = accuracy_score(Y_test, Y_test_predicted)
nn_test_bal_acc = balanced_accuracy_score(Y_test, Y_test_predicted)
print('Neural network acc on test set: {0} \n'.format(
np.round(nn_test_acc, 2)))
print('Neural network bal acc on test set: {0} \n'.format(
np.round(nn_test_bal_acc, 2)))
def train(args, io):
train_loader = DataLoader(ModelNet40(partition='train', num_points=args.num_points), num_workers=8,
batch_size=args.batch_size, shuffle=True, drop_last=False)
test_loader = DataLoader(ModelNet40(partition='test', num_points=args.num_points), num_workers=8,
batch_size=args.test_batch_size, shuffle=False, drop_last=False)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
if args.model == 'pointnet':
model = PointNet(args).to(device)
elif args.model == 'dgcnn':
model = DGCNN(args).to(device)
elif args.model == 'TransformerBaseline':
model = DGCNN_Transformer(args).to(device)
elif args.model == 'TemporalTransformer':
model = DGCNN_TemporalTransformer(args).to(device)
elif args.model == 'TemporalTransformer_v2':
model = DGCNN_TemporalTransformer_v2(args).to(device)
elif args.model == 'TemporalTransformer_v3':
model = DGCNN_TemporalTransformer_v3(args).to(device)
elif args.model == 'pi':
model = pi_DGCNN(args).to(device)
elif args.model == 'pi2':
model = pi_DGCNN_v2(args).to(device)
elif args.model == 'pipoint':
model = pipoint_DGCNN(args).to(device)
else:
raise Exception("Not implemented")
print(str(model))
#model = nn.DataParallel(model, device_ids=list(range(3)))
model = nn.DataParallel(model)
print("Let's use", torch.cuda.device_count(), "GPUs!")
# get the number of model parameters
print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))
if args.use_sgd:
print("Use SGD")
optimizer = optim.SGD(model.parameters(), lr=args.lr*100, momentum=args.momentum, weight_decay=1e-4)
else:
print("Use Adam")
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
scheduler = CosineAnnealingLR(optimizer, args.epochs, eta_min=args.lr)
criterion = cal_loss
best_test_acc = 0
if args.model_path:
if os.path.isfile(args.model_path):
print("=> loading checkpoint '{}'".format(args.model_path))
checkpoint = torch.load(args.model_path)
print(checkpoint)
if 'epoch' in checkpoint:
args.start_epoch = checkpoint['epoch']
#best_prec1 = checkpoint['best_prec1']
#best_prec5 = checkpoint['best_prec5']
if 'state_dict' in checkpoint:
model.load_state_dict(checkpoint['state_dict'])
else:
model.load_state_dict(checkpoint, strict=False)
if 'optimizer' in checkpoint:
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.model_path, args.start_epoch))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
end = time.time()
for epoch in range(args.start_epoch, args.epochs):
scheduler.step()
####################
# Train
####################
train_loss = 0.0
count = 0.0
model.train()
train_pred = []
train_true = []
batch_time = AverageMeter()
data_time = AverageMeter()
end = time.time()
for i, (data, label) in enumerate(train_loader):
data_time.update(time.time()-end)
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
optimizer.zero_grad()
if args.model in ["pi", "pipoint", "pi2"]:
logits, atts = model(data)
elif args.model in ["TransformerBaseline", "TemporalTransformer", "TemporalTransformer_v2", "TemporalTransformer_v3"]:
logits = model(data)
else:
logits, degree = model(data)
'''
if args.visualize == True:
print(args.visualize)
import matplotlib.pyplot as plt
#cmap = plt.cm.get_cmap("hsv", 30)
cmap = plt.cm.get_cmap("binary", 40)
cmap = np.array([cmap(i) for i in range(40)])[:,:3]
obj = degree[7,:,:3].cpu().numpy()
obj_degree = degree[7,:,3:].squeeze()
obj_degree = obj_degree.cpu().numpy().astype(int)
obj_max = np.max(obj_degree)
obj_min = np.min(obj_degree)
print(obj_max)
print(obj_min)
for i in range(obj_min, obj_max):
print("{} : {}".format(i, sum(obj_degree == i)))
gt = cmap[obj_degree-obj_min, :]
showpoints(obj, gt, gt, ballradius=3)
'''
loss = criterion(logits, label, smoothing=True)
loss.backward()
optimizer.step()
preds = logits.max(dim=1)[1]
count += batch_size
train_loss += loss.item() * batch_size
train_true.append(label.cpu().numpy())
train_pred.append(preds.detach().cpu().numpy())
batch_time.update(time.time()-end)
end=time.time()
if i % 10 == 0:
print_str = 'Train {}, loss {}, Time {batch_time.val:.3f} ({batch_time.avg:.3f}), Data {data_time.val:.3f} ({data_time.avg:.3f})'.format(epoch, train_loss*1.0/count, batch_time=batch_time, data_time=data_time)
print(print_str)
train_true = np.concatenate(train_true)
train_pred = np.concatenate(train_pred)
outstr = 'Train %d, loss: %.6f, train acc: %.6f, train avg acc: %.6f' % (epoch,
train_loss*1.0/count,
metrics.accuracy_score(
train_true, train_pred),
metrics.balanced_accuracy_score(
train_true, train_pred))
#outstr = 'Train {}, Time {batch_time.val:.3f} ({batch_time.avg:.3f}), Data {data_time.val:.3f} ({data_time.avg:.3f}), loss: {}, train acc: {}, train avg acc: {}'.format(epoch, batch_time=batch_time, data_time=data_time, train_loss*1.0/count, metrics.accuracy_score(train_true, train_pred), metrics.balanced_accuracy_score(train_true, train_pred))
io.cprint(outstr)
####################
# Test
####################
with torch.no_grad():
test_loss = 0.0
count = 0.0
model.eval()
test_pred = []
test_true = []
batch_time = AverageMeter()
losses = AverageMeter()
end = time.time()
for j, (data, label) in enumerate(test_loader):
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
if args.model in ["pi", "pipoint", "pi2"]:
logits, atts = model(data)
elif args.model in ["TransformerBaseline", "TemporalTransformer", "TemporalTransformer_v2", "TemporalTransformer_v3"]:
logits = model(data)
else:
logits, degree = model(data)
loss = criterion(logits, label, smoothing=True)
preds = logits.max(dim=1)[1]
count += batch_size
test_loss += loss.item() * batch_size
test_true.append(label.cpu().numpy())
test_pred.append(preds.detach().cpu().numpy())
batch_time.update(time.time() - end)
end = time.time()
if j % 10 == 0:
print('Test {}, Loss {}, Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format(j, test_loss*1.0/count, batch_time=batch_time))
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
test_acc = metrics.accuracy_score(test_true, test_pred)
avg_per_class_acc = metrics.balanced_accuracy_score(test_true, test_pred)
#per_class_acc = metrics.precision_score(test_true, test_pred, average=None)
#outstr = 'Test {}, loss: {}, train acc: {}, train avg acc: {}'.format(epoch, batch_time=batch_time, data_time=data_time, test_loss*1.0/count, test_acc, avg_per_class_acc)
outstr = 'Test %d, loss: %.6f, test acc: %.6f, test avg acc: %.6f' % (epoch,
test_loss*1.0/count,
test_acc,
avg_per_class_acc)
#outstr_2 = 'Test per class acc: {}'%per_class_acc
io.cprint(outstr)
#io.cprint(outstr_2)
if args.model in ["pi", "pipoint", "pi2"]:
for j in range(4):
io.cprint('Att {} : {}'.format(j, atts[j].mean().item()))
is_best = test_acc >= best_test_acc
if is_best:
best_test_acc = test_acc
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict(),
}, is_best, args.exp_name)
#ETAPA 03: PREDIÇÃO
Y_maio_19_pred_lgbm = classifier_lgbm.predict(X_maio_19)
#Análise de Métricas
from sklearn.metrics import accuracy_score, balanced_accuracy_score, recall_score, precision_score, f1_score
#Accuracy Score
mtrc_accuracy_score_lgbm = accuracy_score(Y_maio_19, Y_maio_19_pred_lgbm)
print('Accuracy Score : ' + str(mtrc_accuracy_score_lgbm))
#Balanced Accuracy
mtrc_balanced_accuracy_score_lgbm = balanced_accuracy_score(Y_maio_19, Y_maio_19_pred_lgbm)
print('Balanced Accuracy Score : ' + str(mtrc_balanced_accuracy_score_lgbm))
#Precision Score
#Número de vezes que uma classe foi predita corretamente dividida pelo número de vezes que a classe foi predita.
mtrc_precision_score_lgbm = precision_score(Y_maio_19, Y_maio_19_pred_lgbm)
print('Precision Score : ' + str(mtrc_precision_score_lgbm))
#Recall Score
#Número de vezes que uma classe foi predita corretamente (TP) dividido pelo número de vezes que a classe aparece no dado de teste (TP + FN).
mtrc_recall_score_lgbm = recall_score(Y_maio_19, Y_maio_19_pred_lgbm)
print('Recall Score : ' + str(mtrc_recall_score_lgbm))
#F1 Score
def train_CNN(X, epochs, lr):
pred = CNN_model(X)
pred_classes = tf.argmax(pred, axis=1)
labels = tf.argmax(y, axis=1)
# give weight
'''
ratio = 1/10
class_weight = tf.constant([ratio, 1-ratio])
weighted_pred = tf.multiply(pred, class_weight)
'''
# Error function and Optimizer
with tf.name_scope('Error'):
error = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits= pred, labels= y))
tf.summary.scalar('cross_entropy', error)
with tf.name_scope('Optimizer'):
optimizer = tf.train.AdamOptimizer(lr).minimize(error)
# Tensorflow Session
with tf.Session() as sess:
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./Visualization', sess.graph)
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
epoch_loss = 0
start = 0
# Training step
while start < len(y_train):
end = start + batch_size
#X_batch = X_train[start:end]
#y_batch = y_train[start:end]
X_batch, y_batch = next_batch_balanced(batch_size, X_train, y_train)
start += batch_size
# Running training
_, cost = sess.run([optimizer, error], feed_dict= {X: X_batch, y: y_batch, keep_prob: 0.6})
epoch_loss += cost
s = sess.run(merged, feed_dict={X: X_val, y: y_val, keep_prob: 1})
writer.add_summary(s, epoch+1)
print('Training:\nEpoch', epoch+1, 'completed out of', n_epochs, 'loss:', epoch_loss)
y_true= sess.run(labels,feed_dict={X: X_val, y: y_val, keep_prob: 1})
y_pred= sess.run(pred_classes, feed_dict={X: X_val, y: y_val, keep_prob: 1})
print('Validation')
print("Balanced Accuracy: {:.4f}".format(balanced_accuracy_score(y_true, y_pred)))
print("Precision (positive): {:.4f}".format(precision_score(y_true, y_pred, pos_label=1)))
print("Precision (negative): {:.4f}".format(precision_score(y_true, y_pred, pos_label=0)))
print("Recall (positive): {:.4f}".format(recall_score(y_true, y_pred, pos_label=1)))
print("Recall (negative): {:.4f}".format(recall_score(y_true, y_pred, pos_label=0)))
# Testing
y_true = sess.run(labels,feed_dict={X: X_test, y: y_test, keep_prob: 1})
y_pred = sess.run(pred_classes, feed_dict={X: X_test, y: y_test, keep_prob: 1})
print('Testing')
print("Balanced Accuracy: {:.4f}".format(balanced_accuracy_score(y_true, y_pred)))
print("Precision (positive): {:.4f}".format(precision_score(y_true, y_pred, pos_label=1)))
print("Precision (negative): {:.4f}".format(precision_score(y_true, y_pred, pos_label=0)))
print("Recall (positive): {:.4f}".format(recall_score(y_true, y_pred, pos_label=1)))
print("Recall (negative): {:.4f}".format(recall_score(y_true, y_pred, pos_label=0)))
#file.write(testing_string)
saver = tf.train.Saver()
saver.save(sess, './models/dnn', global_step = 1)
writer.close()
def run(self):
np.seterr(divide='warn', invalid='warn')
"""
# Runs the model on the validation set in order to fit a probabilistic model that will evaluate the accuracy of future predictions
"""
output_columns = self.transaction.lmd['predict_columns']
input_columns = [
col for col in self.transaction.lmd['columns']
if col not in output_columns
and col not in self.transaction.lmd['columns_to_ignore']
]
# Make predictions on the validation dataset normally and with various columns missing
normal_predictions = self.transaction.model_backend.predict('validate')
normal_predictions_test = self.transaction.model_backend.predict(
'test')
normal_accuracy = evaluate_accuracy(
normal_predictions,
self.transaction.input_data.validation_df,
self.transaction.lmd['stats_v2'],
output_columns,
backend=self.transaction.model_backend)
for col in output_columns:
if self.transaction.lmd['tss']['is_timeseries']:
reals = list(self.transaction.input_data.validation_df[
self.transaction.input_data.
validation_df['make_predictions'] == True][col])
else:
reals = self.transaction.input_data.validation_df[col]
preds = normal_predictions[col]
fails = False
data_type = self.transaction.lmd['stats_v2'][col]['typing'][
'data_type']
data_subtype = self.transaction.lmd['stats_v2'][col]['typing'][
'data_subtype']
if data_type == DATA_TYPES.CATEGORICAL:
if data_subtype == DATA_SUBTYPES.TAGS:
encoder = self.transaction.model_backend.predictor._mixer.encoders[
col]
if balanced_accuracy_score(
encoder.encode(reals).argmax(axis=1),
encoder.encode(preds).argmax(
axis=1)) <= self.transaction.lmd['stats_v2'][
col]['balanced_guess_probability']:
fails = True
else:
if balanced_accuracy_score(
reals, preds) <= self.transaction.lmd['stats_v2'][
col]['balanced_guess_probability']:
fails = True
elif data_type == DATA_TYPES.NUMERIC:
if r2_score(reals, preds) < 0:
fails = True
else:
pass
if fails:
if not self.transaction.lmd['force_predict']:
def predict_wrapper(*args, **kwargs):
raise Exception('Failed to train model')
self.session.predict = predict_wrapper
log.error('Failed to train model to predict {}'.format(col))
empty_input_predictions = {}
empty_input_accuracy = {}
empty_input_predictions_test = {}
ignorable_input_columns = [
x for x in input_columns if self.transaction.lmd['stats_v2'][x]
['typing']['data_type'] != DATA_TYPES.FILE_PATH and (
not self.transaction.lmd['tss']['is_timeseries']
or x not in self.transaction.lmd['tss']['order_by'])
]
for col in ignorable_input_columns:
empty_input_predictions[
col] = self.transaction.model_backend.predict(
'validate', ignore_columns=[col])
empty_input_predictions_test[
col] = self.transaction.model_backend.predict(
'test', ignore_columns=[col])
empty_input_accuracy[col] = evaluate_accuracy(
empty_input_predictions[col],
self.transaction.input_data.validation_df,
self.transaction.lmd['stats_v2'],
output_columns,
backend=self.transaction.model_backend)
# Get some information about the importance of each column
self.transaction.lmd['column_importances'] = {}
for col in ignorable_input_columns:
accuracy_increase = (normal_accuracy - empty_input_accuracy[col])
# normalize from 0 to 10
self.transaction.lmd['column_importances'][col] = 10 * max(
0, accuracy_increase)
# Run Probabilistic Validator
overall_accuracy_arr = []
self.transaction.lmd['accuracy_histogram'] = {}
self.transaction.lmd['confusion_matrices'] = {}
self.transaction.lmd['accuracy_samples'] = {}
self.transaction.hmd['probabilistic_validators'] = {}
self.transaction.lmd['train_data_accuracy'] = {}
self.transaction.lmd['test_data_accuracy'] = {}
self.transaction.lmd['valid_data_accuracy'] = {}
for col in output_columns:
# Training data accuracy
predictions = self.transaction.model_backend.predict(
'predict_on_train_data',
ignore_columns=self.transaction.lmd['stats_v2']
['columns_to_ignore'])
self.transaction.lmd['train_data_accuracy'][
col] = evaluate_accuracy(
predictions,
self.transaction.input_data.train_df,
self.transaction.lmd['stats_v2'], [col],
backend=self.transaction.model_backend)
# Testing data accuracy
predictions = self.transaction.model_backend.predict(
'test',
ignore_columns=self.transaction.lmd['stats_v2']
['columns_to_ignore'])
self.transaction.lmd['test_data_accuracy'][
col] = evaluate_accuracy(
predictions,
self.transaction.input_data.test_df,
self.transaction.lmd['stats_v2'], [col],
backend=self.transaction.model_backend)
# Validation data accuracy
predictions = self.transaction.model_backend.predict(
'validate',
ignore_columns=self.transaction.lmd['stats_v2']
['columns_to_ignore'])
self.transaction.lmd['valid_data_accuracy'][
col] = evaluate_accuracy(
predictions,
self.transaction.input_data.validation_df,
self.transaction.lmd['stats_v2'], [col],
backend=self.transaction.model_backend)
for col in output_columns:
pval = ProbabilisticValidator(
col_stats=self.transaction.lmd['stats_v2'][col],
col_name=col,
input_columns=input_columns)
predictions_arr = [normal_predictions_test] + [
x for x in empty_input_predictions_test.values()
]
pval.fit(self.transaction.input_data.test_df, predictions_arr,
[[ignored_column]
for ignored_column in empty_input_predictions_test])
overall_accuracy, accuracy_histogram, cm, accuracy_samples = pval.get_accuracy_stats(
)
overall_accuracy_arr.append(overall_accuracy)
self.transaction.lmd['accuracy_histogram'][
col] = accuracy_histogram
self.transaction.lmd['confusion_matrices'][col] = cm
self.transaction.lmd['accuracy_samples'][col] = accuracy_samples
self.transaction.hmd['probabilistic_validators'][col] = pickle_obj(
pval)
self.transaction.lmd['validation_set_accuracy'] = sum(
overall_accuracy_arr) / len(overall_accuracy_arr)
# conformal prediction confidence estimation
self.transaction.lmd['stats_v2']['train_std_dev'] = {}
self.transaction.hmd['label_encoders'] = {}
self.transaction.hmd['icp'] = {'active': False}
for target in output_columns:
data_type = self.transaction.lmd['stats_v2'][target]['typing'][
'data_type']
data_subtype = self.transaction.lmd['stats_v2'][target]['typing'][
'data_subtype']
is_classification = data_type == DATA_TYPES.CATEGORICAL
fit_params = {
'target': target,
'all_columns': self.transaction.lmd['columns'],
'columns_to_ignore': []
}
fit_params['columns_to_ignore'].extend(
self.transaction.lmd['columns_to_ignore'])
fit_params['columns_to_ignore'].extend(
[col for col in output_columns if col != target])
if is_classification:
if data_subtype != DATA_SUBTYPES.TAGS:
all_targets = [
elt[1][target].values for elt in inspect.getmembers(
self.transaction.input_data)
if elt[0] in {'test_df', 'train_df', 'validation_df'}
]
all_classes = np.unique(
np.concatenate([np.unique(arr)
for arr in all_targets]))
enc = OneHotEncoder(sparse=False, handle_unknown='ignore')
enc.fit(all_classes.reshape(-1, 1))
fit_params['one_hot_enc'] = enc
self.transaction.hmd['label_encoders'][target] = enc
else:
fit_params['one_hot_enc'] = None
self.transaction.hmd['label_encoders'][target] = None
adapter = ConformalClassifierAdapter
nc_function = MarginErrFunc(
) # better than IPS as we'd need the complete distribution over all classes
nc_class = ClassifierNc
icp_class = IcpClassifier
else:
adapter = ConformalRegressorAdapter
nc_function = AbsErrorErrFunc()
nc_class = RegressorNc
icp_class = IcpRegressor
if (data_type == DATA_TYPES.NUMERIC or
(is_classification and data_subtype != DATA_SUBTYPES.TAGS)
) and not self.transaction.lmd['tss']['is_timeseries']:
model = adapter(self.transaction.model_backend.predictor,
fit_params=fit_params)
nc = nc_class(model, nc_function)
X = deepcopy(self.transaction.input_data.train_df)
y = X.pop(target)
if is_classification:
self.transaction.hmd['icp'][target] = icp_class(
nc, smoothing=False)
else:
self.transaction.hmd['icp'][target] = icp_class(nc)
self.transaction.lmd['stats_v2']['train_std_dev'][
target] = self.transaction.input_data.train_df[
target].std()
X = clean_df(X, self.transaction.lmd['stats_v2'],
output_columns)
self.transaction.hmd['icp'][target].fit(X.values, y.values)
self.transaction.hmd['icp']['active'] = True
# calibrate conformal estimator on test set
X = deepcopy(self.transaction.input_data.validation_df)
y = X.pop(target).values
if is_classification:
if isinstance(enc.categories_[0][0], str):
cats = enc.categories_[0].tolist()
y = np.array([cats.index(i) for i in y])
y = y.astype(int)
X = clean_df(X, self.transaction.lmd['stats_v2'],
output_columns)
self.transaction.hmd['icp'][target].calibrate(X.values, y)
def dlModel_classifier(X_train,
y_train,
X_test,
y_test,
EPOCHS=100,
batch_size=5,
loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=SGD(lr=0.01, momentum=0.9)):
start_time = time.time()
estimator = DLmodel_baseline(X_train, y_train, X_test, y_test, loss,
metrics, optimizer)
# Validation: ¶
# Fit the model
history = estimator.fit(X_train,
y_train,
validation_split=0.20,
epochs=180,
batch_size=10,
verbose=0)
# list all data in history
print(history.history.keys())
#estimator.fit(X_train, y_train, epochs=EPOCHS, validation_split=0.2, verbose=1, batch_size=batch_size)
y_pred = estimator.predict_classes(X_test)
# summarizing historical accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
time_end = time.time() - start_time
# Scores
acc = accuracy_score(y_pred=y_pred, y_true=y_test) * 100
print('accuracy ' + str(acc))
print('balanced_accuracy_score ' +
str(balanced_accuracy_score(y_pred=y_pred, y_true=y_test) * 100))
#f1_score
f1 = f1_score(y_pred=y_pred, y_true=y_test, average='macro') * 100
print('f1_score ' + str(f1))
#precision_score
prec = precision_score(y_pred=y_pred, y_true=y_test,
average='weighted') * 100
print('precision_score ' + str(prec))
#log_loss
print('Log loss ' + str(log_loss(y_pred=y_pred, y_true=y_test)))
#recall_score
recall = recall_score(y_pred=y_pred, y_true=y_test) * 100
print('recall_score ' + str(recall))
#roc_curve
y_pred_proba = estimator.predict_proba(X_test)
false_positive_rate, true_positive_rate, thresholds = roc_curve(
y_true=y_test, y_score=y_pred_proba)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(false_positive_rate, true_positive_rate, label="Keras")
plt.xlabel('false_positive_rate')
plt.ylabel('true_positive_rate')
plt.title('ROC curve of ' + "Keras")
plt.show()
#roc_auc_score
roc = roc_auc_score(y_test, y_pred_proba) * 100
print('roc_auc_score ' + str(roc))
#confusion_matrix
print('confusion_matrix ' + str(confusion_matrix(y_test, y_pred)))
class_report = classification_report(y_test, y_pred)
print("classification_report" + str(class_report))
f = open(mlresult_dir + str(project_identifier) + "_log_dlModels.csv", "a")
f.write("\n Time taken to execute the Keras is " + str(time_end) + "\n" +
"Dated on" + str(datetime.datetime.now()) + "\n" +
' Accuracy Score' + str(acc) + "\n" + 'f1_score ' + str(f1) +
"\n" + 'precision_score ' + str(prec) + "\n" + 'recall_score ' +
str(recall) + "\n" + 'roc_auc_score ' + str(roc) + "\n" +
'classification_report ' + str(class_report))
f.close()
print("\n Time taken to execute the Keras is " + str(time_end))
print("\n" + "Dated on" + str(datetime.datetime.now()) + "\n")
# resturn model object
return estimator, y_pred
import joblib
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, f1_score, balanced_accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
x_test = joblib.load('data/x_test_w.joblib')
y_test = joblib.load('data/y_test_w.joblib')
loaded = joblib.load('data/white-final-trained-model.joblib')
pred = loaded.predict(x_test)
print('test acc= ', accuracy_score(pred, y_test))
print('bal test acc= ', balanced_accuracy_score(pred, y_test))
print('F1 score= ', f1_score(pred, y_test, average='micro'))
print('F1 score= ', f1_score(pred, y_test, average=None))
cm = confusion_matrix(y_test, pred)
display = ConfusionMatrixDisplay(cm).plot()
plt.figure(1)
plt.title('Test white wine')
classNames = ['poor', 'average', 'excellent']
tick_marks = np.arange(len(classNames))
plt.xticks(tick_marks, classNames)
plt.yticks(tick_marks, classNames)
plt.show(display)
def infer(valid_queue, model, criterion):
objs = utils.AvgrageMeter()
top1 = utils.AvgrageMeter()
top5 = utils.AvgrageMeter()
model.eval()
preds = np.asarray([])
targets = np.asarray([])
logits_pred = []
names = []
for step, (input, target, name) in enumerate(valid_queue):
#input = input.cuda()
#target = target.cuda(non_blocking=True)
input = Variable(input, volatile=True).cuda()
target = Variable(target, volatile=True).cuda(async=True)
logits = model(input)
loss = criterion(logits, target)
prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5))
n = input.size(0)
#objs.update(loss.data[0], n)
#top1.update(prec1.data[0], n)
#top5.update(prec5.data[0], n)
objs.update(loss.item(), n)
top1.update(prec1.item(), n)
top5.update(prec5.item(), n)
#minha alteracao
output = logits
topk = (1, 3)
maxk = max(topk)
batch_size = target.size(0)
_, predicted = torch.max(output.data, 1)
#minha alteracao
preds = np.concatenate((preds, predicted.cpu().numpy().ravel()))
#targets = np.concatenate((targets,target.cpu().numpy().ravel()))
targets = np.concatenate((targets, target.data.cpu().numpy().ravel()))
names.append(name)
logits_pred.append(output.data.cpu().numpy())
if step % args.report_freq == 0:
logging.info('valid %03d %e %f %f', step, objs.avg, top1.avg,
top5.avg)
print(preds.shape)
print(targets.shape)
print('np.unique(targets):', np.unique(targets))
print('np.unique(preds): ', np.unique(preds))
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
cr = classification_report(targets, preds, output_dict=True)
a1, a2, a3 = cr['macro avg']['f1-score'], cr['macro avg']['precision'], cr[
'macro avg']['recall']
topover = (a1 + a2 + a3) / 3
print(classification_report(targets, preds))
from sklearn.metrics import balanced_accuracy_score
from sklearn.metrics import accuracy_score
print(balanced_accuracy_score(targets, preds))
acc = accuracy_score(targets, preds)
log_score = open("log_score.txt", "a")
log_score.write('logits: ' + str(logits_pred) + "\n")
log_score.write('names: ' + str(names) + "\n")
log_score.write('accuracy:' + str(acc) + '\n')
log_score.write('report: ' + str(cr) + '\n')
log_score.close()
print("SAVED!!")
from sklearn.metrics import confusion_matrix
matrix = confusion_matrix(targets, preds)
print(matrix.diagonal() / matrix.sum(axis=1))
print(matrix)
return top1.avg, objs.avg
def bmac(self, clf, x_test, y_test):
from sklearn.metrics import balanced_accuracy_score
return balanced_accuracy_score(y_test, self.clf.predict(x_test))
# Classification using a single decision tree
###############################################################################
# We train a decision tree classifier which will be used as a baseline for the
# rest of this example.
###############################################################################
# The results are reported in terms of balanced accuracy and geometric mean
# which are metrics widely used in the literature to validate model trained on
# imbalanced set.
tree = DecisionTreeClassifier()
tree.fit(X_train, y_train)
y_pred_tree = tree.predict(X_test)
print('Decision tree classifier performance:')
print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'
.format(balanced_accuracy_score(y_test, y_pred_tree),
geometric_mean_score(y_test, y_pred_tree)))
cm_tree = confusion_matrix(y_test, y_pred_tree)
fig, ax = plt.subplots()
plot_confusion_matrix(cm_tree, classes=np.unique(satimage.target), ax=ax,
title='Decision tree')
###############################################################################
# Classification using bagging classifier with and without sampling
###############################################################################
# Instead of using a single tree, we will check if an ensemble of decsion tree
# can actually alleviate the issue induced by the class imbalancing. First, we
# will use a bagging classifier and its counter part which internally uses a
# random under-sampling to balanced each boostrap sample.
bagging = BaggingClassifier(n_estimators=50, random_state=0, n_jobs=-1)
def gini_bal_accuracy(truth, predictions):
try:
from sklearn.metrics import balanced_accuracy_score
return balanced_accuracy_score(truth, predictions)
except:
return accuracy_score(truth, predictions)
