def evaluate(model, acoustic_iterator, linguistic_iterator, criterion):
model.eval()
epoch_loss = 0
conf_mat = np.zeros((4, 4))
assert len(acoustic_iterator) == len(linguistic_iterator)
with torch.no_grad():
for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(),
linguistic_iterator()):
acoustic_batch = acoustic_tuple[0]
acoustic_labels = acoustic_tuple[1]
linguistic_batch = linguistic_tuple[0]
linguistic_labels = linguistic_tuple[1]
predictions = model(acoustic_batch, linguistic_batch).squeeze(1)
loss = criterion(predictions.float(), acoustic_labels)
epoch_loss += loss.item()
conf_mat += confusion_matrix(predictions, acoustic_labels)
acc = sum([conf_mat[i, i]
for i in range(conf_mat.shape[0])]) / conf_mat.sum()
acc_per_class = [
conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0])
]
weighted_acc = sum(acc_per_class) / len(acc_per_class)
return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
def confusion_matrix_age_filter(age_min, age_max, test,
predicted_OUTPUT_with_age):
count = 0
for i, row in test.iterrows():
if (row['AGE'] < age_min or row['AGE'] > age_max
) and predicted_OUTPUT_with_age[count] == 1:
predicted_OUTPUT_with_age[count] = 0
count += 1
cnf_matrix = metrics.confusion_matrix(test.OUTPUT,
predicted_OUTPUT_with_age)
class_names = [0, 1]
fig, ax = cnf_plt.subplots()
tick_marks = np.arange(len(class_names))
cnf_plt.xticks(tick_marks, class_names, fontsize=14)
cnf_plt.yticks(tick_marks, class_names, fontsize=14)
# create heatmap
sns.heatmap(pd.DataFrame(cnf_matrix),
annot=True,
annot_kws={"size": 22},
cmap="YlGnBu",
fmt='g')
ax.xaxis.set_label_position("top")
cnf_plt.tight_layout()
cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14)
cnf_plt.ylabel('Actual label', fontsize=14)
cnf_plt.xlabel('Predicted label', fontsize=14)
conf_matrix_fig = cnf_plt.gcf()
conf_matrix_fig.tight_layout()
cnf_plt.draw()
conf_matrix_fig.savefig('plots/conf_matrix_age_{}_to_{}.png'.format(
age_min, age_max))
cnf_plt.clf()
def _evaluations(predictions, labels, num_classes, class_names):
"""Returns a group of operations to calculate accuracy and other evaluation metrics."""
ops = [
metrics.accuracy(predictions, labels, num_classes),
metrics.confusion_matrix(predictions, labels, num_classes),
metrics.class_metrics(predictions, labels, class_names)
]
return tf.group(*ops)
def test(classifier,x_test,y_test):
prediction = classifier.predict(x_test)
print("Confusion Matrix for Decision tree Classofier Model given : ")
print(confusion_matrix(y_test,prediction))
print("Classification Report for given Decision tree Classisifer : ")
print(classification_report(y_test,prediction))
print("Accuracy Score : ",accuracy_score(y_test,prediction))
return
def test_metrics():
y_true = [0, 1, 1, 0, 1, 1]
y_pred = [0, 0, 1, 0, 0, 1]
# metrics + cm counts
observed_metrics = precision_recall_f1_accuracy(y_true, y_pred, label=1)
# Confusion Matrix - counts and normalized
observed_normalized_cm = confusion_matrix(y_true,
y_pred,
label=1,
normalize=True)
observed_cm = confusion_matrix(y_true, y_pred, label=1, normalize=False)
expected_metrics = {
"precision": 1.00,
"recall": 0.50,
"f1_score": 0.67,
"accuracy": 66.67,
"TP": 2,
"FN": 2,
"FP": 0,
"TN": 2,
}
expected_normalized_cm = [[1.0, 0.0], [0.5, 0.5]]
expected_cm = [[2, 0], [2, 2]]
assert observed_metrics == expected_metrics
assert observed_cm == expected_cm
assert observed_normalized_cm == expected_normalized_cm
actual = ["B", "B", "C", "A", "B", "B", "C", "A", "A", "B", "D"]
predicted = ["A", "B", "B", "A", "C", "B", "C", "C", "A", "C", "D"]
print_normalized_confusion_matrix(actual, predicted)
matrix, labels = confusion_matrix(actual, predicted, normalize=True)
assert matrix == [
[0.67, 0.33, 0.0, 0.0],
[0.0, 0.67, 0.33, 0.0],
[0.25, 0.5, 0.25, 0.0],
[0.0, 0.0, 0.0, 1.0],
]
assert sorted(labels) == ["A", "B", "C", "D"]
compute_all(actual, predicted)
def test_1(self): actual = [1, 1, 0, 1, 1, 1, 0, 0, 1, 1] predicted = [0, 1, 0, 1, 0, 1, 0, 1, 0, 0] tp, fn, fp, tn = metrics.confusion_matrix(actual, predicted) self.assertEqual(tp, 3) self.assertEqual(fn, 4) self.assertEqual(fp, 1) self.assertEqual(tn, 2) self.assertEqual(metrics.accuracy(actual, predicted), 0.5) self.assertEqual(metrics.precision(actual, predicted), 3/4) self.assertEqual(metrics.recall(actual, predicted), 3/7) self.assertEqual(metrics.f1(actual, predicted), 6/11)
def metrics_calculator(masks, preds, mode_average=True, additional=False):
batch_size, masks, predictions = mtr.standardize_for_metrics(masks, preds)
accuracy_score = mtr.accuracy(batch_size, masks, predictions, mode_average)
if additional:
roc_auc_score = mtr.roc_auc(batch_size, masks, predictions,
mode_average)
jaccard_score = mtr.jaccard(batch_size, masks, predictions,
mode_average)
sens_score, spec_score, prec_score, f1_score = mtr.confusion_matrix(
batch_size, masks, predictions, mode_average)
pr_auc_score = mtr.precision_recall_auc(batch_size, masks, predictions,
mode_average)
iou_score = mtr.fast_hist(predictions, masks, 2)
return roc_auc_score, accuracy_score, jaccard_score, sens_score, spec_score, prec_score, f1_score, pr_auc_score, iou_score
return accuracy_score
def train(model, acoustic_iterator, linguistic_iterator, optimizer, criterion,
reg_ratio):
model.train()
epoch_loss = 0
conf_mat = np.zeros((4, 4))
assert len(acoustic_iterator) == len(linguistic_iterator)
for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(),
linguistic_iterator()):
acoustic_batch = acoustic_tuple[0]
acoustic_labels = acoustic_tuple[1]
linguistic_batch = linguistic_tuple[0]
linguistic_labels = linguistic_tuple[1]
optimizer.zero_grad()
predictions = model(acoustic_batch, linguistic_batch).squeeze(1)
loss = criterion(predictions, acoustic_labels)
reg_loss = 0
for param in model.parameters():
reg_loss += param.norm(2)
total_loss = loss + reg_ratio * reg_loss
total_loss.backward()
optimizer.step()
epoch_loss += loss.item()
conf_mat += confusion_matrix(predictions, acoustic_labels)
acc = sum([conf_mat[i, i]
for i in range(conf_mat.shape[0])]) / conf_mat.sum()
acc_per_class = [
conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0])
]
weighted_acc = sum(acc_per_class) / len(acc_per_class)
return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
def main():
# check for gpu device
print(tf.test.gpu_device_name())
# print tf and keras version
print(tf.VERSION)
print(tf.keras.__version__)
data, labels = data_prep.read_images("data/train",
IMAGE_DIMS,
general=True)
print("Train Set loaded")
data_test, labels_test = data_prep.read_images("data/test",
IMAGE_DIMS,
general=True)
print("Test Set loaded")
lb = LabelBinarizer()
X_train, y_train = data_prep.binarize(data, labels, lb)
X_test, y_test = data_prep.binarize(data_test, labels_test, lb)
print(X_train.shape)
print(y_train.shape)
model = VGGNet.vgg_net(N_CLASSES, IMAGE_DIMS)
# Train the model
start = time.time()
history = VGGNet.fit(model, X_train, y_train, X_test, y_test, EPOCHS, BS)
end = time.time()
print("Training Time: " + timer(start, end))
# Metrics
metrics.plot_evaluation(history)
y_pred = model.predict(X_test)
classification_report = metrics.classification_report(y_test, y_pred, lb)
print("Classification report : \n", classification_report)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
print("Confusion Matrix : \n", confusion_matrix)
metrics.print_confusion_matrix(y_test, y_pred, lb)
def confusion_matrix_multiple_diseases(test,
predicted_OUTPUT_multiple_diseases,
number_of_diseases):
count = 0
for i, row in test.iterrows():
if (len(row['DIAGNOSIS'].split(';'))
) >= number_of_diseases and predicted_OUTPUT_multiple_diseases[
count] == 1:
predicted_OUTPUT_multiple_diseases[count] = 0
count += 1
cnf_matrix = metrics.confusion_matrix(test.OUTPUT,
predicted_OUTPUT_multiple_diseases)
class_names = [0, 1]
fig, ax = cnf_plt.subplots()
tick_marks = np.arange(len(class_names))
cnf_plt.xticks(tick_marks, class_names, fontsize=14)
cnf_plt.yticks(tick_marks, class_names, fontsize=14)
# create heatmap
sns.heatmap(pd.DataFrame(cnf_matrix),
annot=True,
annot_kws={"size": 22},
cmap="YlGnBu",
fmt='g')
ax.xaxis.set_label_position("top")
cnf_plt.tight_layout()
cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14)
cnf_plt.ylabel('Actual label', fontsize=14)
cnf_plt.xlabel('Predicted label', fontsize=14)
conf_matrix_fig = cnf_plt.gcf()
conf_matrix_fig.tight_layout()
cnf_plt.draw()
conf_matrix_fig.savefig(
'plots/conf_matrix_{}_diseases_dismissed.png'.format(
number_of_diseases))
cnf_plt.clf()
train_X, test_X = data.iloc[train_index], data.iloc[test_index]
train_y, test_y = y.iloc[train_index], y.iloc[test_index]
clf = None
clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
max_depth=max_depth, min_samples_split=min_samples_split,
n_jobs=n_processes)
clf.fit(train_X, train_y)
results.append(metrics.predict_table(clf, test_X, test_y))
ids.extend(test_X.index.tolist())
if validation == 'holdout':
aux = metrics.predict_table(clf, test_X, test_y)
aux.to_csv(result_path + 'Predicciones/hold_' + str(count) + '.csv')
print 'hold ' + str(count) + ' ' + str(metrics.weighted_f_score(metrics.confusion_matrix(aux)))
count += 1
result = pd.concat(results)
result['indice'] = ids
result.set_index('indice')
result.index.name = catalog + '_id'
result = result.drop('indice', axis=1)
output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+')
pickle.dump(clf, output)
output.close()
result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv')
print metrics.weighted_f_score(metrics.confusion_matrix(result))
index_filter = args.index_filter
feature_filter = args.feature_filter
if index_filter is not None:
index_filter = pd.read_csv(index_filter, index_col=0).index
paths = [sets_path + catalog + '_sampled_' + str(i) + '.csv' for i in xrange(n_samples)]
if model == 'tree':
partial_fit = partial(parallel.fit_tree, feature_filter=feature_filter, folds=folds,
inverse=inverse, max_depth=max_depth,
min_samples_split=min_samples_split, lc_filter=lc_filter)
elif model == 'rf':
partial_fit = partial(parallel.fit_rf, feature_filter=feature_filter, folds=folds,
inverse=inverse, lc_filter=lc_filter)
elif model == 'sktree':
partial_fit = partial(parallel.fit_sktree, feature_filter=feature_filter, folds=folds,
inverse=inverse, max_depth=max_depth,
min_samples_split=min_samples_split, lc_filter=lc_filter)
pool = Pool(processes=n_processes, maxtasksperchild=2)
resultados = pool.map(partial_fit, paths)
pool.close()
pool.join()
result = metrics.aggregate_predictions(resultados)
result.to_csv(result_path)
print metrics.weighted_f_score(metrics.confusion_matrix(result))
ntrain = 6000
nvalid = 1000
nclass = 10
Xtrain,Ytrain,Xvalid,Yvalid = load_mnist.load_mnist(ntrain,nvalid)
# (2) Define model
nfeature = Xtrain.shape[0]
np.random.seed(10)
lamb = 0.0
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(128,"relu",lamb)
model.add_layer(nclass,"softmax",lamb)
# (3) Compile model
optimizer = Optimizer.Adam(0.02,0.9,0.999,1e-7)
model.compile("crossentropy",optimizer)
model.summary()
# (4) Train model
epochs = 40
time_start = time.time()
history = model.fit(Xtrain,Ytrain,epochs,batch_size=ntrain,validation_data=(Xvalid,Yvalid))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Predictions and plotting
# confusion matrix
print("Metrics for Validation Dataset")
Yvalid_pred = model.predict(Xvalid)
metrics.confusion_matrix(Yvalid,Yvalid_pred,nclass)
# plot loss, accuracy, and animation of results
plot_results.plot_results_history(history,["loss","valid_loss"])
plot_results.plot_results_history(history,["accuracy","valid_accuracy"])
plot_results.plot_results_mnist_animation(Xvalid,Yvalid,Yvalid_pred,model.get_Afinal(),100)
plt.show()
# %%
metrics.worst_samples(imgs_valid,
labels_valid,
scores_predict,
class_to_index,
top=16,
names_valid=None)
# %%
metrics.classification_report(labels_valid, label_predict, class_to_index)
# %%
metrics.confusion_matrix(labels_valid, label_predict, class_to_index)
# %%
metrics.ROC(labels_valid, scores_predict, class_to_index, section=[0, 5, 11])
#%%
metrics.ROCCompare(labels_valids,
scores_predicts,
class_to_index,
subtitles,
section=[0, 5])
# %%
features, targets, fraction)
btrainf = np.ceil(np.asarray(trainf) / abs(np.asarray(trainf)))
btestf = np.ceil(np.asarray(testf) / abs(np.asarray(testf)))
# btrainf = btrainf.tolist()
# btestf = btestf.tolist()
if model == "decision_tree":
tree = DecisionTree(attribute_names)
tree.fit(btrainf, traint)
num_nodes, max_depth = tree.tree_attributes(tree.tree)
t = tree.predict(btestf)
decision = tree.predict(today)
#print(decision)
#print("Should you buy tomorrow with today's data? %d" % decision)
cm = metrics.confusion_matrix(testt, t)
a = metrics.accuracy(testt, t)
p, r = metrics.precision_and_recall(testt, t)
f = metrics.f1_measure(testt, t)
print("Accuracy = %f\n" % a)
print("Precision = %f, Recall = %f\n" % (p, r))
print("F1 measure = %f\n" % f)
print("Tomorrow's Forecast: %f\n" % decision[-1])
elif model == "prior_probability":
prob = PriorProbability()
print(btrainf)
prob.fit(btrainf, traint)
t = prob.predict(btestf)
decision = prob.predict(today)
#raise ValueError(t)
def evaluate(self,
experiment_path: Path,
task: str = 'aurora_clean',
model_resolution=0.02,
time_resolution=0.02,
threshold=(0.5, 0.1),
**kwargs):
EVALUATION_DATA = {
'aurora_clean': {
'data': 'data/evaluation/hdf5/aurora_clean.h5',
'label': 'data/evaluation/labels/aurora_clean_labels.tsv',
},
'aurora_noisy': {
'data': 'data/evaluation/hdf5/aurora_noisy.h5',
'label': 'data/evaluation/labels/aurora_noisy_labels.tsv'
},
'dihard_dev': {
'data': 'data/evaluation/hdf5/dihard_dev.h5',
'label': 'data/evaluation/labels/dihard_dev.csv'
},
'dihard_eval': {
'data': 'data/evaluation/hdf5/dihard_eval.h5',
'label': 'data/evaluation/labels/dihard_eval.csv'
},
'aurora_snr_20': {
'data':
'data/evaluation/hdf5/aurora_noisy_musan_snr_20.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'aurora_snr_15': {
'data':
'data/evaluation/hdf5/aurora_noisy_musan_snr_15.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'aurora_snr_10': {
'data':
'data/evaluation/hdf5/aurora_noisy_musan_snr_10.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'aurora_snr_5': {
'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_5.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'aurora_snr_0': {
'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_0.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'aurora_snr_-5': {
'data':
'data/evaluation/hdf5/aurora_noisy_musan_snr_-5.0.hdf5',
'label': 'data/evaluation/labels/musan_labels.tsv'
},
'dcase18': {
'data': 'data/evaluation/hdf5/dcase18.h5',
'label': 'data/evaluation/labels/dcase18.tsv',
},
}
assert task in EVALUATION_DATA, f"--task {'|'.join(list(EVALUATION_DATA.keys()))}"
experiment_path = Path(experiment_path)
if experiment_path.is_file(): # Model is given
model_path = experiment_path
experiment_path = experiment_path.parent
else:
model_path = next(Path(experiment_path).glob("run_model*"))
config = torch.load(next(Path(experiment_path).glob("run_config*")),
map_location='cpu')
logger = utils.getfile_outlogger(None)
# Use previous config, but update data such as kwargs
config_parameters = dict(config, **kwargs)
# Default columns to search for in data
model_parameters = torch.load(
model_path, map_location=lambda storage, loc: storage)
encoder = torch.load('labelencoders/vad.pth')
data = EVALUATION_DATA[task]['data']
label_df = pd.read_csv(EVALUATION_DATA[task]['label'], sep='\s+')
label_df['filename'] = label_df['filename'].apply(
lambda x: Path(x).name)
logger.info(f"Label_df shape is {label_df.shape}")
dset = dataset.EvalH5Dataset(data,
fnames=np.unique(
label_df['filename'].values))
dataloader = torch.utils.data.DataLoader(dset,
batch_size=1,
num_workers=4,
shuffle=False)
model = getattr(models, config_parameters['model'])(
inputdim=dataloader.dataset.datadim,
outputdim=len(encoder.classes_),
**config_parameters['model_args'])
model.load_state_dict(model_parameters)
model = model.to(DEVICE).eval()
## VAD preprocessing data
vad_label_helper_df = label_df.copy()
vad_label_helper_df['onset'] = np.ceil(vad_label_helper_df['onset'] /
model_resolution).astype(int)
vad_label_helper_df['offset'] = np.ceil(vad_label_helper_df['offset'] /
model_resolution).astype(int)
vad_label_helper_df = vad_label_helper_df.groupby(['filename']).agg({
'onset':
tuple,
'offset':
tuple,
'event_label':
tuple
}).reset_index()
logger.trace(model)
output_dfs = []
speech_label_idx = np.where('Speech' == encoder.classes_)[0].squeeze()
speech_frame_predictions, speech_frame_ground_truth, speech_frame_prob_predictions = [], [],[]
# Using only binary thresholding without filter
if len(threshold) == 1:
postprocessing_method = utils.binarize
else:
postprocessing_method = utils.double_threshold
with torch.no_grad(), tqdm(total=len(dataloader),
leave=False,
unit='clip') as pbar:
for feature, filename in dataloader:
feature = torch.as_tensor(feature).to(DEVICE)
# PANNS output a dict instead of 2 values
prediction_tag, prediction_time = model(feature)
prediction_tag = prediction_tag.to('cpu')
prediction_time = prediction_time.to('cpu')
if prediction_time is not None: # Some models do not predict timestamps
cur_filename = filename[0]
thresholded_prediction = postprocessing_method(
prediction_time, *threshold)
## VAD predictions
speech_frame_prob_predictions.append(
prediction_time[..., speech_label_idx].squeeze())
### Thresholded speech predictions
speech_prediction = thresholded_prediction[
..., speech_label_idx].squeeze()
speech_frame_predictions.append(speech_prediction)
targets = vad_label_helper_df[
vad_label_helper_df['filename'] == cur_filename][[
'onset', 'offset'
]].values[0]
target_arr = np.zeros_like(speech_prediction)
for start, end in zip(*targets):
target_arr[start:end] = 1
speech_frame_ground_truth.append(target_arr)
#### SED predictions
labelled_predictions = utils.decode_with_timestamps(
encoder, thresholded_prediction)
pred_label_df = pd.DataFrame(
labelled_predictions[0],
columns=['event_label', 'onset', 'offset'])
if not pred_label_df.empty:
pred_label_df['filename'] = cur_filename
pred_label_df['onset'] *= model_resolution
pred_label_df['offset'] *= model_resolution
pbar.set_postfix(labels=','.join(
np.unique(pred_label_df['event_label'].values)))
pbar.update()
output_dfs.append(pred_label_df)
full_prediction_df = pd.concat(output_dfs)
prediction_df = full_prediction_df[full_prediction_df['event_label'] ==
'Speech']
assert set(['onset', 'offset', 'filename', 'event_label'
]).issubset(prediction_df.columns), "Format is wrong"
assert set(['onset', 'offset', 'filename', 'event_label'
]).issubset(label_df.columns), "Format is wrong"
logger.info("Calculating VAD measures ... ")
speech_frame_ground_truth = np.concatenate(speech_frame_ground_truth,
axis=0)
speech_frame_predictions = np.concatenate(speech_frame_predictions,
axis=0)
speech_frame_prob_predictions = np.concatenate(
speech_frame_prob_predictions, axis=0)
vad_results = []
tn, fp, fn, tp = metrics.confusion_matrix(
speech_frame_ground_truth, speech_frame_predictions).ravel()
fer = 100 * ((fp + fn) / len(speech_frame_ground_truth))
acc = 100 * ((tp + tn) / (len(speech_frame_ground_truth)))
p_miss = 100 * (fn / (fn + tp))
p_fa = 100 * (fp / (fp + tn))
for i in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.7,0.9]:
mp_fa, mp_miss = metrics.obtain_error_rates(
speech_frame_ground_truth, speech_frame_prob_predictions, i)
tn, fp, fn, tp = metrics.confusion_matrix(
speech_frame_ground_truth,
speech_frame_prob_predictions > i).ravel()
sub_fer = 100 * ((fp + fn) / len(speech_frame_ground_truth))
logger.info(
f"PFa {100*mp_fa:.2f} Pmiss {100*mp_miss:.2f} FER {sub_fer:.2f} t: {i:.2f}"
)
auc = metrics.roc(speech_frame_ground_truth,
speech_frame_prob_predictions) * 100
for avgtype in ('micro', 'macro', 'binary'):
precision, recall, f1, _ = metrics.precision_recall_fscore_support(
speech_frame_ground_truth,
speech_frame_predictions,
average=avgtype)
vad_results.append(
(avgtype, 100 * precision, 100 * recall, 100 * f1))
logger.info("Calculating segment based metric .. ")
# Change order just for better printing in file
prediction_df = prediction_df[[
'filename', 'onset', 'offset', 'event_label'
]]
metric = metrics.segment_based_evaluation_df(
label_df, prediction_df, time_resolution=time_resolution)
logger.info("Calculating event based metric .. ")
event_metric = metrics.event_based_evaluation_df(
label_df, prediction_df)
prediction_df.to_csv(experiment_path /
f'speech_predictions_{task}.tsv',
sep='\t',
index=False)
full_prediction_df.to_csv(experiment_path / f'predictions_{task}.tsv',
sep='\t',
index=False)
with open(experiment_path / f'evaluation_{task}.txt', 'w') as fp:
for k, v in config_parameters.items():
print(f"{k}:{v}", file=fp)
print(metric, file=fp)
print(event_metric, file=fp)
for avgtype, precision, recall, f1 in vad_results:
print(
f"VAD {avgtype} F1: {f1:<10.3f} {precision:<10.3f} Recall: {recall:<10.3f}",
file=fp)
print(f"FER: {fer:.2f}", file=fp)
print(f"AUC: {auc:.2f}", file=fp)
print(f"Pfa: {p_fa:.2f}", file=fp)
print(f"Pmiss: {p_miss:.2f}", file=fp)
print(f"ACC: {acc:.2f}", file=fp)
logger.info(f"Results are at {experiment_path}")
for avgtype, precision, recall, f1 in vad_results:
print(
f"VAD {avgtype:<10} F1: {f1:<10.3f} Pre: {precision:<10.3f} Recall: {recall:<10.3f}"
)
print(f"FER: {fer:.2f}")
print(f"AUC: {auc:.2f}")
print(f"Pfa: {p_fa:.2f}")
print(f"Pmiss: {p_miss:.2f}")
print(f"ACC: {acc:.2f}")
print(event_metric)
print(metric)
print("XvalidT.shape: {}".format(XvalidT.shape))
print("YvalidT.shape: {}".format(YvalidT.shape))
# (2) Define model
nfeature = Xtrain.shape[0]
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(200,input_shape=(nfeature,),activation="tanh"),
tf.keras.layers.Dense(50,activation="tanh"),
tf.keras.layers.Dense(1,activation="sigmoid")])
# (3) Compile model
optimizer = tf.keras.optimizers.Adam(lr=0.02,beta_1=0.9,beta_2=0.999,epsilon=1e-7)
model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=["accuracy"])
model.summary()
# (4) Train model
epochs = 20
time_start = time.time()
ntrain = XtrainT.shape[0]
history = model.fit(XtrainT,YtrainT,epochs=epochs,batch_size=ntrain,validation_data=(XvalidT,YvalidT))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Predictions and plotting
# confusion matrix
Afinal = model.predict(XvalidT).T
Yvalid_pred = np.round(Afinal)
metrics.confusion_matrix(Yvalid,Yvalid_pred,2)
f1score,precision,recall = metrics.f1score(Yvalid,Yvalid_pred)
print("F1Score: {} - Precision: {} - Recall: {}".format(f1score,precision,recall))
text_results.text_results(Yvalid,Yvalid_pred,Xvalid_raw)
# plot loss and accuracy
plot_results.plot_results_history(history.history,["loss","val_loss"])
plot_results.plot_results_history(history.history,["accuracy","val_accuracy"])
plt.show()
def evaluate(self, actual: np.array, predicted: np.array) -> confusion_matrix:
return confusion_matrix(actual, predicted, labels=[0,1])
train_X, test_X = data.iloc[train_index], data.iloc[test_index]
train_y, test_y = y.iloc[train_index], y.iloc[test_index]
clf = None
clf = RandomForestClassifier(n_estimators=p,
criterion='entropy',
max_depth=14,
min_samples_split=20,
n_jobs=2)
clf.fit(train_X, train_y)
results.append(metrics.predict_table(clf, test_X, test_y))
result = pd.concat(results)
matrix = metrics.confusion_matrix(result)
clases = matrix.columns.tolist()
precisions = [metrics.precision(matrix, c) for c in clases]
recalls = [metrics.recall(matrix, c) for c in clases]
f_scores = [metrics.f_score(matrix, c) for c in clases]
w_score = metrics.weighted_f_score(matrix)
# f = open(result_dir + str(max_depth) + ' ' + str(min_samples_split) + '.txt', 'w')
f = open(result_dir + str(p) + '.txt', 'w')
f.write('F_score by class')
f.write('\n')
f.write(str(f_scores))
f.write('\n')
def valid(model, validation_loader, loss, best_loss, pad_size, orig_dim,
num_img):
# set the model to be ready to test on the validation set
model.eval()
validation_acc = 0.0
validation_loss = 0.0
validation_IoU = 0.0
IoU_count = 0.0
# test the model
print("Testing model...", flush=True)
# run through the batches of validation images
for validation_imgs, validation_labels in validation_loader:
# use GPU for images if possible
if torch.cuda.is_available():
validation_imgs = validation_imgs.cuda()
validation_labels = validation_labels.cuda()
# predict the classes (whether a pixel contains a building or not)
outputs = model(validation_imgs)
# compute the loss
loss_output = loss(outputs, validation_labels.long())
# calculate the validation loss and find the prediction for each pixel of an image
validation_loss += loss_output.item() * validation_imgs.size(0)
_, prediction = torch.max(outputs.data, 1)
# if the image and label were padded beforehand crop them back into their original
# dimensions
if pad_size != 0:
prediction = crop(prediction.float(), orig_dim)
validation_labels = crop(validation_labels, orig_dim)
for img in range(validation_labels.shape[0]):
# set up the confusion matrix and calculate the metrics
# check that there is an object to compute the metrics with
if torch.sum(validation_labels[img]) != 0:
IoU_count += 1
_, _, IoU = confusion_matrix(
np.asarray(prediction[img].cpu()),
np.asarray(validation_labels[img].cpu()), orig_dim)
validation_IoU += IoU
# calculate the accuracy after each batch
validation_acc += torch.sum(
prediction.long() == validation_labels.long().data)
# calculate the total accuracy, validation loss, and IoU
validation_acc = validation_acc.item() / (float(num_img) * orig_dim *
orig_dim)
validation_loss = validation_loss / num_img
# make sure there is no divide by 0
if IoU_count != 0:
validation_IoU = validation_IoU / IoU_count
print("Val acc: " + str(validation_acc),
"Val loss: " + str(validation_loss),
flush=True)
print("Val IoU: " + str(validation_IoU), flush=True)
# save the model's learned parameters with the lowest loss
if best_loss > validation_loss:
best_loss = validation_loss
save_model(model)
return best_loss
feature_filter = args.feature_filter
train_data = pd.read_csv(training_set_path, index_col=0)
train_X, train_y = utils.filter_data(train_data, feature_filter=feature_filter)
test_data = pd.read_csv(test_set_path, index_col=0)
test_X, test_y = utils.filter_data(test_data, feature_filter=feature_filter)
clf = None
clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
max_depth=max_depth, min_samples_split=min_samples_split,
n_jobs=n_processes)
clf.fit(train_X, train_y)
result = metrics.predict_table(clf, test_X, test_y)
result['indice'] = test_X.index.tolist()
result.set_index('indice')
result.index.name = catalog + '_id'
result = result.drop('indice', axis=1)
output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+')
pickle.dump(clf, output)
output.close()
result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv')
print metrics.weighted_f_score(metrics.confusion_matrix(result))
# bikin confusion matrix dan metric dari training ini:
prediction = model.predict(X_test)
label_pred = []
for i in range(prediction.shape[1]):
label_pred.append(np.argmax(prediction[:, i]))
y_test_label = []
for i in range(y_test.shape[0]):
y_test_label.append(np.argmax(y_test[i, :]))
metrics = Metrics(y_test_label, label_pred)
print("ACCURACY SLURRR: ", metrics.all_accuracy())
print("PRECISION SLURRR: ", metrics.all_precision())
print("RECALL SLURRR: ", metrics.all_recall())
print("F1 SLURRR: ", metrics.all_f1_score())
print("CONFUSION MATRIXX: ")
print(confusion_matrix(y_test_label, label_pred))
print("--------------------------------------")
# 10-fold cross validation
k_fold = KFold(n_splits=10)
print(k_fold)
scores = []
for train_index, test_index in k_fold.split(X):
print("TRAIN DATA INDEX")
print(train_index)
print("TEST DATA INDEX")
print(test_index)
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
results = []
skf = cross_validation.StratifiedKFold(y, n_folds=folds)
for train_index, test_index in skf:
train_X, test_X = data.iloc[train_index], data.iloc[test_index]
train_y, test_y = y.iloc[train_index], y.iloc[test_index]
clf = None
clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
max_depth=max_depth, min_samples_split=min_samples_split,
n_jobs=n_processes)
clf.fit(train_X, train_y)
results.append(metrics.predict_table(clf, test_X, test_y))
result = pd.concat(results)
output = open(result_path + 'Arboles/Arbol.pkl', 'wb+')
pickle.dump(clf, output)
output.close()
result.to_csv(result_path + 'Predicciones/result.csv')
matrix = metrics.confusion_matrix(result)
matrix.to_csv(result_path + 'Metricas/soft_matrix_.csv')
clases = matrix.columns.tolist()
f_score = [metrics.f_score(matrix, c) for c in clases]
with open(result_path + 'Metricas/results.txt') as f:
f.write(clases + '\n')
f.write(str(f_score) + '\n')
def validate(self, dataloader, is_val_set=True, measure_entropy=True):
self.net.eval()
self.hook.flag_hook = True
prefix = self.get_prefix(is_val_set)
self.metrics_epoch = collections.defaultdict(utils.Meter)
matrix = np.zeros((self.c_dim, self.c_dim), dtype=np.uint32)
for mb, (x, y) in enumerate(dataloader):
x, y = x.to(self.device), y.to(self.device)
y_one_hot = utils.to_one_hot(y, self.c_dim)
logits = self.net(x)
losses = self.criterion(logits, y_one_hot)
matrix = matrix + metrics.confusion_matrix(
utils.tensor2array(utils.get_class_outputs(logits)),
utils.tensor2array(y), self.c_dim)
self.metrics_epoch['{}_loss'.format(prefix)].update(
utils.tensor2array(losses), x.shape[0])
if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
num_log = min(self.cfg.num_log, x.shape[0])
name = '{}_{}_{}_epoch{:0=3d}_minibatch{}'
filepath = '{}/{}'.format(
os.path.join(self.cfg.plot_dir, self.cfg.model_type,
self.cfg.model_name), name)
x_ = x[0:num_log]
x_np, y_np = utils.tensor2array(
x[0:num_log]), utils.tensor2array(y[0:num_log])
losses_np = utils.tensor2array(losses[0:num_log])
plotting.make_grid(
x_,
filepath.format(prefix, 'data', 'x',
self.metrics['epochs'][-1], mb + 1))
utils.save_array(
x_np,
filepath.format(prefix, 'data', 'x',
self.metrics['epochs'][-1], mb + 1))
utils.save_array(
y_np,
filepath.format(prefix, 'data', 'y',
self.metrics['epochs'][-1], mb + 1))
utils.save_array(
losses_np,
filepath.format(prefix, 'data', 'losses',
self.metrics['epochs'][-1], mb + 1))
for (k, layer_name) in enumerate(self.hook.layers):
layer_np = utils.tensor2array(
self.hook.layers[layer_name][0:num_log])
utils.save_array(
layer_np,
filepath.format(prefix, 'data', layer_name,
self.metrics['epochs'][-1], mb + 1))
if measure_entropy:
entropy = metrics.entropy(utils.logits_to_probs(logits))
self.metrics_epoch['{}_entropy'.format(prefix)].update(
utils.tensor2array(entropy), x.shape[0])
if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
entropy_np = utils.tensor2array(entropy[0:num_log])
utils.save_array(
entropy_np,
filepath.format(prefix, 'data', 'entropy',
self.metrics['epochs'][-1], mb + 1))
if is_val_set:
x_rand = torch.randn(size=x.shape).to(self.device)
logits_rand = self.net(x_rand)
entropy_rand = metrics.entropy(
utils.logits_to_probs(logits_rand))
self.metrics_epoch['entropy_rand'].update(
utils.tensor2array(entropy_rand), x.shape[0])
if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
name = '{}_{}_{}_epoch{:0=3d}_minibatch{}'
filepath = '{}/{}'.format(
os.path.join(self.cfg.plot_dir,
self.cfg.model_type,
self.cfg.model_name), name)
x_ = x_rand[0:num_log]
x_np = utils.tensor2array(x_rand[0:num_log])
entropy_np = utils.tensor2array(
entropy_rand[0:num_log])
plotting.make_grid(
x_,
filepath.format(prefix, 'noise', 'x',
self.metrics['epochs'][-1],
mb + 1))
utils.save_array(
x_np,
filepath.format(prefix, 'noise', 'x',
self.metrics['epochs'][-1],
mb + 1))
utils.save_array(
entropy_np,
filepath.format(prefix, 'noise', 'entropy',
self.metrics['epochs'][-1],
mb + 1))
for (k, layer_name) in enumerate(self.hook.layers):
layer_np = utils.tensor2array(
self.hook.layers[layer_name][0:num_log])
utils.save_array(
layer_np,
filepath.format(prefix, 'noise', layer_name,
self.metrics['epochs'][-1],
mb + 1))
# disable hook after first minibatch by default - this is done for computational/speed purposes
self.hook.flag_hook = False
self.summarize_metrics(matrix, prefix)