def evaluate(model, acoustic_iterator, linguistic_iterator, criterion):
    model.eval()

    epoch_loss = 0
    conf_mat = np.zeros((4, 4))

    assert len(acoustic_iterator) == len(linguistic_iterator)

    with torch.no_grad():
        for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(),
                                                    linguistic_iterator()):
            acoustic_batch = acoustic_tuple[0]
            acoustic_labels = acoustic_tuple[1]
            linguistic_batch = linguistic_tuple[0]
            linguistic_labels = linguistic_tuple[1]
            predictions = model(acoustic_batch, linguistic_batch).squeeze(1)

            loss = criterion(predictions.float(), acoustic_labels)
            epoch_loss += loss.item()
            conf_mat += confusion_matrix(predictions, acoustic_labels)

    acc = sum([conf_mat[i, i]
               for i in range(conf_mat.shape[0])]) / conf_mat.sum()
    acc_per_class = [
        conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0])
    ]
    weighted_acc = sum(acc_per_class) / len(acc_per_class)

    return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
Пример #2
0
def confusion_matrix_age_filter(age_min, age_max, test,
                                predicted_OUTPUT_with_age):
    count = 0
    for i, row in test.iterrows():
        if (row['AGE'] < age_min or row['AGE'] > age_max
            ) and predicted_OUTPUT_with_age[count] == 1:
            predicted_OUTPUT_with_age[count] = 0
        count += 1

    cnf_matrix = metrics.confusion_matrix(test.OUTPUT,
                                          predicted_OUTPUT_with_age)

    class_names = [0, 1]
    fig, ax = cnf_plt.subplots()
    tick_marks = np.arange(len(class_names))
    cnf_plt.xticks(tick_marks, class_names, fontsize=14)
    cnf_plt.yticks(tick_marks, class_names, fontsize=14)

    # create heatmap
    sns.heatmap(pd.DataFrame(cnf_matrix),
                annot=True,
                annot_kws={"size": 22},
                cmap="YlGnBu",
                fmt='g')
    ax.xaxis.set_label_position("top")
    cnf_plt.tight_layout()
    cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14)
    cnf_plt.ylabel('Actual label', fontsize=14)
    cnf_plt.xlabel('Predicted label', fontsize=14)
    conf_matrix_fig = cnf_plt.gcf()
    conf_matrix_fig.tight_layout()
    cnf_plt.draw()
    conf_matrix_fig.savefig('plots/conf_matrix_age_{}_to_{}.png'.format(
        age_min, age_max))
    cnf_plt.clf()
Пример #3
0
def _evaluations(predictions, labels, num_classes, class_names):
    """Returns a group of operations to calculate accuracy and other evaluation metrics."""
    ops = [
        metrics.accuracy(predictions, labels, num_classes),
        metrics.confusion_matrix(predictions, labels, num_classes),
        metrics.class_metrics(predictions, labels, class_names)
    ]
    return tf.group(*ops)
Пример #4
0
def test(classifier,x_test,y_test):
    prediction = classifier.predict(x_test)
    print("Confusion Matrix for Decision tree Classofier Model given : ")
    print(confusion_matrix(y_test,prediction))
    print("Classification Report for given Decision tree Classisifer : ")
    print(classification_report(y_test,prediction))
    print("Accuracy Score  : ",accuracy_score(y_test,prediction))
    return
def test_metrics():
    y_true = [0, 1, 1, 0, 1, 1]
    y_pred = [0, 0, 1, 0, 0, 1]

    # metrics + cm counts
    observed_metrics = precision_recall_f1_accuracy(y_true, y_pred, label=1)
    # Confusion Matrix - counts and normalized
    observed_normalized_cm = confusion_matrix(y_true,
                                              y_pred,
                                              label=1,
                                              normalize=True)
    observed_cm = confusion_matrix(y_true, y_pred, label=1, normalize=False)

    expected_metrics = {
        "precision": 1.00,
        "recall": 0.50,
        "f1_score": 0.67,
        "accuracy": 66.67,
        "TP": 2,
        "FN": 2,
        "FP": 0,
        "TN": 2,
    }
    expected_normalized_cm = [[1.0, 0.0], [0.5, 0.5]]
    expected_cm = [[2, 0], [2, 2]]

    assert observed_metrics == expected_metrics
    assert observed_cm == expected_cm
    assert observed_normalized_cm == expected_normalized_cm

    actual = ["B", "B", "C", "A", "B", "B", "C", "A", "A", "B", "D"]
    predicted = ["A", "B", "B", "A", "C", "B", "C", "C", "A", "C", "D"]
    print_normalized_confusion_matrix(actual, predicted)
    matrix, labels = confusion_matrix(actual, predicted, normalize=True)
    assert matrix == [
        [0.67, 0.33, 0.0, 0.0],
        [0.0, 0.67, 0.33, 0.0],
        [0.25, 0.5, 0.25, 0.0],
        [0.0, 0.0, 0.0, 1.0],
    ]
    assert sorted(labels) == ["A", "B", "C", "D"]

    compute_all(actual, predicted)
Пример #6
0
	def test_1(self):
		actual = [1, 1, 0, 1, 1, 1, 0, 0, 1, 1]
		predicted = [0, 1, 0, 1, 0, 1, 0, 1, 0, 0]
		tp, fn, fp, tn = metrics.confusion_matrix(actual, predicted)
		self.assertEqual(tp, 3)
		self.assertEqual(fn, 4)
		self.assertEqual(fp, 1)
		self.assertEqual(tn, 2)
		self.assertEqual(metrics.accuracy(actual, predicted), 0.5)
		self.assertEqual(metrics.precision(actual, predicted), 3/4)
		self.assertEqual(metrics.recall(actual, predicted), 3/7)
		self.assertEqual(metrics.f1(actual, predicted), 6/11)
Пример #7
0
def metrics_calculator(masks, preds, mode_average=True, additional=False):
    batch_size, masks, predictions = mtr.standardize_for_metrics(masks, preds)
    accuracy_score = mtr.accuracy(batch_size, masks, predictions, mode_average)
    if additional:
        roc_auc_score = mtr.roc_auc(batch_size, masks, predictions,
                                    mode_average)
        jaccard_score = mtr.jaccard(batch_size, masks, predictions,
                                    mode_average)
        sens_score, spec_score, prec_score, f1_score = mtr.confusion_matrix(
            batch_size, masks, predictions, mode_average)
        pr_auc_score = mtr.precision_recall_auc(batch_size, masks, predictions,
                                                mode_average)
        iou_score = mtr.fast_hist(predictions, masks, 2)
        return roc_auc_score, accuracy_score, jaccard_score, sens_score, spec_score, prec_score, f1_score, pr_auc_score, iou_score
    return accuracy_score
def train(model, acoustic_iterator, linguistic_iterator, optimizer, criterion,
          reg_ratio):
    model.train()

    epoch_loss = 0
    conf_mat = np.zeros((4, 4))

    assert len(acoustic_iterator) == len(linguistic_iterator)

    for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(),
                                                linguistic_iterator()):
        acoustic_batch = acoustic_tuple[0]
        acoustic_labels = acoustic_tuple[1]
        linguistic_batch = linguistic_tuple[0]
        linguistic_labels = linguistic_tuple[1]
        optimizer.zero_grad()

        predictions = model(acoustic_batch, linguistic_batch).squeeze(1)

        loss = criterion(predictions, acoustic_labels)

        reg_loss = 0
        for param in model.parameters():
            reg_loss += param.norm(2)

        total_loss = loss + reg_ratio * reg_loss
        total_loss.backward()

        optimizer.step()

        epoch_loss += loss.item()

        conf_mat += confusion_matrix(predictions, acoustic_labels)

    acc = sum([conf_mat[i, i]
               for i in range(conf_mat.shape[0])]) / conf_mat.sum()
    acc_per_class = [
        conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0])
    ]
    weighted_acc = sum(acc_per_class) / len(acc_per_class)

    return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
Пример #9
0
def main():
    # check for gpu device
    print(tf.test.gpu_device_name())
    # print tf and keras version
    print(tf.VERSION)
    print(tf.keras.__version__)

    data, labels = data_prep.read_images("data/train",
                                         IMAGE_DIMS,
                                         general=True)
    print("Train Set loaded")
    data_test, labels_test = data_prep.read_images("data/test",
                                                   IMAGE_DIMS,
                                                   general=True)
    print("Test Set loaded")

    lb = LabelBinarizer()
    X_train, y_train = data_prep.binarize(data, labels, lb)
    X_test, y_test = data_prep.binarize(data_test, labels_test, lb)
    print(X_train.shape)
    print(y_train.shape)

    model = VGGNet.vgg_net(N_CLASSES, IMAGE_DIMS)

    # Train the model
    start = time.time()
    history = VGGNet.fit(model, X_train, y_train, X_test, y_test, EPOCHS, BS)
    end = time.time()
    print("Training Time: " + timer(start, end))

    # Metrics
    metrics.plot_evaluation(history)
    y_pred = model.predict(X_test)
    classification_report = metrics.classification_report(y_test, y_pred, lb)
    print("Classification report : \n", classification_report)
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("Confusion Matrix : \n", confusion_matrix)
    metrics.print_confusion_matrix(y_test, y_pred, lb)
Пример #10
0
def confusion_matrix_multiple_diseases(test,
                                       predicted_OUTPUT_multiple_diseases,
                                       number_of_diseases):
    count = 0
    for i, row in test.iterrows():
        if (len(row['DIAGNOSIS'].split(';'))
            ) >= number_of_diseases and predicted_OUTPUT_multiple_diseases[
                count] == 1:
            predicted_OUTPUT_multiple_diseases[count] = 0
        count += 1

    cnf_matrix = metrics.confusion_matrix(test.OUTPUT,
                                          predicted_OUTPUT_multiple_diseases)

    class_names = [0, 1]
    fig, ax = cnf_plt.subplots()
    tick_marks = np.arange(len(class_names))
    cnf_plt.xticks(tick_marks, class_names, fontsize=14)
    cnf_plt.yticks(tick_marks, class_names, fontsize=14)

    # create heatmap
    sns.heatmap(pd.DataFrame(cnf_matrix),
                annot=True,
                annot_kws={"size": 22},
                cmap="YlGnBu",
                fmt='g')
    ax.xaxis.set_label_position("top")
    cnf_plt.tight_layout()
    cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14)
    cnf_plt.ylabel('Actual label', fontsize=14)
    cnf_plt.xlabel('Predicted label', fontsize=14)
    conf_matrix_fig = cnf_plt.gcf()
    conf_matrix_fig.tight_layout()
    cnf_plt.draw()
    conf_matrix_fig.savefig(
        'plots/conf_matrix_{}_diseases_dismissed.png'.format(
            number_of_diseases))
    cnf_plt.clf()
Пример #11
0
        train_X, test_X = data.iloc[train_index], data.iloc[test_index]
        train_y, test_y = y.iloc[train_index], y.iloc[test_index]

        clf = None
        clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
                                     max_depth=max_depth, min_samples_split=min_samples_split,
                                     n_jobs=n_processes)

        clf.fit(train_X, train_y)
        results.append(metrics.predict_table(clf, test_X, test_y))
        ids.extend(test_X.index.tolist())

        if validation == 'holdout':
            aux = metrics.predict_table(clf, test_X, test_y)
            aux.to_csv(result_path + 'Predicciones/hold_' + str(count) + '.csv')
            print 'hold ' + str(count) + ' ' + str(metrics.weighted_f_score(metrics.confusion_matrix(aux)))
            count += 1

    result = pd.concat(results)
    result['indice'] = ids
    result.set_index('indice')
    result.index.name = catalog + '_id'
    result = result.drop('indice', axis=1)

    output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+')
    pickle.dump(clf, output)
    output.close()

    result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv')

    print metrics.weighted_f_score(metrics.confusion_matrix(result))
Пример #12
0
    index_filter = args.index_filter
    feature_filter = args.feature_filter

    if index_filter is not None:
        index_filter = pd.read_csv(index_filter, index_col=0).index

    paths = [sets_path + catalog + '_sampled_' + str(i) + '.csv' for i in xrange(n_samples)]

    if model == 'tree':
        partial_fit = partial(parallel.fit_tree, feature_filter=feature_filter, folds=folds,
                              inverse=inverse, max_depth=max_depth,
                              min_samples_split=min_samples_split, lc_filter=lc_filter)
    elif model == 'rf':
        partial_fit = partial(parallel.fit_rf, feature_filter=feature_filter, folds=folds,
                              inverse=inverse, lc_filter=lc_filter)
    elif model == 'sktree':
        partial_fit = partial(parallel.fit_sktree, feature_filter=feature_filter, folds=folds,
                              inverse=inverse, max_depth=max_depth,
                              min_samples_split=min_samples_split, lc_filter=lc_filter)

    pool = Pool(processes=n_processes, maxtasksperchild=2)
    
    resultados = pool.map(partial_fit, paths)
    pool.close()
    pool.join()

    result = metrics.aggregate_predictions(resultados)
    result.to_csv(result_path)

    print metrics.weighted_f_score(metrics.confusion_matrix(result))
ntrain = 6000
nvalid = 1000
nclass = 10
Xtrain,Ytrain,Xvalid,Yvalid = load_mnist.load_mnist(ntrain,nvalid)
# (2) Define model
nfeature = Xtrain.shape[0]
np.random.seed(10)
lamb = 0.0
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(128,"relu",lamb)
model.add_layer(nclass,"softmax",lamb)
# (3) Compile model
optimizer = Optimizer.Adam(0.02,0.9,0.999,1e-7)
model.compile("crossentropy",optimizer)
model.summary()
# (4) Train model
epochs = 40
time_start = time.time()
history = model.fit(Xtrain,Ytrain,epochs,batch_size=ntrain,validation_data=(Xvalid,Yvalid))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Predictions and plotting
# confusion matrix
print("Metrics for Validation Dataset")
Yvalid_pred = model.predict(Xvalid)
metrics.confusion_matrix(Yvalid,Yvalid_pred,nclass)
# plot loss, accuracy, and animation of results
plot_results.plot_results_history(history,["loss","valid_loss"])
plot_results.plot_results_history(history,["accuracy","valid_accuracy"])
plot_results.plot_results_mnist_animation(Xvalid,Yvalid,Yvalid_pred,model.get_Afinal(),100)
plt.show()
Пример #14
0
# %%

metrics.worst_samples(imgs_valid,
                      labels_valid,
                      scores_predict,
                      class_to_index,
                      top=16,
                      names_valid=None)

# %%

metrics.classification_report(labels_valid, label_predict, class_to_index)

# %%

metrics.confusion_matrix(labels_valid, label_predict, class_to_index)

# %%

metrics.ROC(labels_valid, scores_predict, class_to_index, section=[0, 5, 11])

#%%

metrics.ROCCompare(labels_valids,
                   scores_predicts,
                   class_to_index,
                   subtitles,
                   section=[0, 5])

# %%
    features, targets, fraction)
btrainf = np.ceil(np.asarray(trainf) / abs(np.asarray(trainf)))
btestf = np.ceil(np.asarray(testf) / abs(np.asarray(testf)))

# btrainf = btrainf.tolist()
# btestf = btestf.tolist()

if model == "decision_tree":
    tree = DecisionTree(attribute_names)
    tree.fit(btrainf, traint)
    num_nodes, max_depth = tree.tree_attributes(tree.tree)
    t = tree.predict(btestf)
    decision = tree.predict(today)
    #print(decision)
    #print("Should you buy tomorrow with today's data? %d" % decision)
    cm = metrics.confusion_matrix(testt, t)
    a = metrics.accuracy(testt, t)
    p, r = metrics.precision_and_recall(testt, t)
    f = metrics.f1_measure(testt, t)
    print("Accuracy = %f\n" % a)
    print("Precision = %f, Recall = %f\n" % (p, r))
    print("F1 measure = %f\n" % f)
    print("Tomorrow's Forecast: %f\n" % decision[-1])

elif model == "prior_probability":
    prob = PriorProbability()
    print(btrainf)
    prob.fit(btrainf, traint)
    t = prob.predict(btestf)
    decision = prob.predict(today)
    #raise ValueError(t)
Пример #16
0
    def evaluate(self,
                 experiment_path: Path,
                 task: str = 'aurora_clean',
                 model_resolution=0.02,
                 time_resolution=0.02,
                 threshold=(0.5, 0.1),
                 **kwargs):
        EVALUATION_DATA = {
            'aurora_clean': {
                'data': 'data/evaluation/hdf5/aurora_clean.h5',
                'label': 'data/evaluation/labels/aurora_clean_labels.tsv',
            },
            'aurora_noisy': {
                'data': 'data/evaluation/hdf5/aurora_noisy.h5',
                'label': 'data/evaluation/labels/aurora_noisy_labels.tsv'
            },
            'dihard_dev': {
                'data': 'data/evaluation/hdf5/dihard_dev.h5',
                'label': 'data/evaluation/labels/dihard_dev.csv'
            },
            'dihard_eval': {
                'data': 'data/evaluation/hdf5/dihard_eval.h5',
                'label': 'data/evaluation/labels/dihard_eval.csv'
            },
            'aurora_snr_20': {
                'data':
                'data/evaluation/hdf5/aurora_noisy_musan_snr_20.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'aurora_snr_15': {
                'data':
                'data/evaluation/hdf5/aurora_noisy_musan_snr_15.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'aurora_snr_10': {
                'data':
                'data/evaluation/hdf5/aurora_noisy_musan_snr_10.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'aurora_snr_5': {
                'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_5.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'aurora_snr_0': {
                'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_0.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'aurora_snr_-5': {
                'data':
                'data/evaluation/hdf5/aurora_noisy_musan_snr_-5.0.hdf5',
                'label': 'data/evaluation/labels/musan_labels.tsv'
            },
            'dcase18': {
                'data': 'data/evaluation/hdf5/dcase18.h5',
                'label': 'data/evaluation/labels/dcase18.tsv',
            },
        }
        assert task in EVALUATION_DATA, f"--task {'|'.join(list(EVALUATION_DATA.keys()))}"
        experiment_path = Path(experiment_path)
        if experiment_path.is_file():  # Model is given
            model_path = experiment_path
            experiment_path = experiment_path.parent
        else:
            model_path = next(Path(experiment_path).glob("run_model*"))
        config = torch.load(next(Path(experiment_path).glob("run_config*")),
                            map_location='cpu')
        logger = utils.getfile_outlogger(None)
        # Use previous config, but update data such as kwargs
        config_parameters = dict(config, **kwargs)
        # Default columns to search for in data
        model_parameters = torch.load(
            model_path, map_location=lambda storage, loc: storage)
        encoder = torch.load('labelencoders/vad.pth')
        data = EVALUATION_DATA[task]['data']
        label_df = pd.read_csv(EVALUATION_DATA[task]['label'], sep='\s+')
        label_df['filename'] = label_df['filename'].apply(
            lambda x: Path(x).name)
        logger.info(f"Label_df shape is {label_df.shape}")

        dset = dataset.EvalH5Dataset(data,
                                     fnames=np.unique(
                                         label_df['filename'].values))

        dataloader = torch.utils.data.DataLoader(dset,
                                                 batch_size=1,
                                                 num_workers=4,
                                                 shuffle=False)

        model = getattr(models, config_parameters['model'])(
            inputdim=dataloader.dataset.datadim,
            outputdim=len(encoder.classes_),
            **config_parameters['model_args'])

        model.load_state_dict(model_parameters)
        model = model.to(DEVICE).eval()

        ## VAD preprocessing data
        vad_label_helper_df = label_df.copy()
        vad_label_helper_df['onset'] = np.ceil(vad_label_helper_df['onset'] /
                                               model_resolution).astype(int)
        vad_label_helper_df['offset'] = np.ceil(vad_label_helper_df['offset'] /
                                                model_resolution).astype(int)

        vad_label_helper_df = vad_label_helper_df.groupby(['filename']).agg({
            'onset':
            tuple,
            'offset':
            tuple,
            'event_label':
            tuple
        }).reset_index()
        logger.trace(model)

        output_dfs = []

        speech_label_idx = np.where('Speech' == encoder.classes_)[0].squeeze()
        speech_frame_predictions, speech_frame_ground_truth, speech_frame_prob_predictions = [], [],[]
        # Using only binary thresholding without filter
        if len(threshold) == 1:
            postprocessing_method = utils.binarize
        else:
            postprocessing_method = utils.double_threshold
        with torch.no_grad(), tqdm(total=len(dataloader),
                                   leave=False,
                                   unit='clip') as pbar:
            for feature, filename in dataloader:
                feature = torch.as_tensor(feature).to(DEVICE)
                # PANNS output a dict instead of 2 values
                prediction_tag, prediction_time = model(feature)
                prediction_tag = prediction_tag.to('cpu')
                prediction_time = prediction_time.to('cpu')

                if prediction_time is not None:  # Some models do not predict timestamps

                    cur_filename = filename[0]

                    thresholded_prediction = postprocessing_method(
                        prediction_time, *threshold)

                    ## VAD predictions
                    speech_frame_prob_predictions.append(
                        prediction_time[..., speech_label_idx].squeeze())
                    ### Thresholded speech predictions
                    speech_prediction = thresholded_prediction[
                        ..., speech_label_idx].squeeze()
                    speech_frame_predictions.append(speech_prediction)
                    targets = vad_label_helper_df[
                        vad_label_helper_df['filename'] == cur_filename][[
                            'onset', 'offset'
                        ]].values[0]
                    target_arr = np.zeros_like(speech_prediction)
                    for start, end in zip(*targets):
                        target_arr[start:end] = 1
                    speech_frame_ground_truth.append(target_arr)

                    #### SED predictions

                    labelled_predictions = utils.decode_with_timestamps(
                        encoder, thresholded_prediction)
                    pred_label_df = pd.DataFrame(
                        labelled_predictions[0],
                        columns=['event_label', 'onset', 'offset'])
                    if not pred_label_df.empty:
                        pred_label_df['filename'] = cur_filename
                        pred_label_df['onset'] *= model_resolution
                        pred_label_df['offset'] *= model_resolution
                        pbar.set_postfix(labels=','.join(
                            np.unique(pred_label_df['event_label'].values)))
                        pbar.update()
                        output_dfs.append(pred_label_df)

        full_prediction_df = pd.concat(output_dfs)
        prediction_df = full_prediction_df[full_prediction_df['event_label'] ==
                                           'Speech']
        assert set(['onset', 'offset', 'filename', 'event_label'
                    ]).issubset(prediction_df.columns), "Format is wrong"
        assert set(['onset', 'offset', 'filename', 'event_label'
                    ]).issubset(label_df.columns), "Format is wrong"
        logger.info("Calculating VAD measures ... ")
        speech_frame_ground_truth = np.concatenate(speech_frame_ground_truth,
                                                   axis=0)
        speech_frame_predictions = np.concatenate(speech_frame_predictions,
                                                  axis=0)
        speech_frame_prob_predictions = np.concatenate(
            speech_frame_prob_predictions, axis=0)

        vad_results = []
        tn, fp, fn, tp = metrics.confusion_matrix(
            speech_frame_ground_truth, speech_frame_predictions).ravel()
        fer = 100 * ((fp + fn) / len(speech_frame_ground_truth))
        acc = 100 * ((tp + tn) / (len(speech_frame_ground_truth)))

        p_miss = 100 * (fn / (fn + tp))
        p_fa = 100 * (fp / (fp + tn))
        for i in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.7,0.9]:
            mp_fa, mp_miss = metrics.obtain_error_rates(
                speech_frame_ground_truth, speech_frame_prob_predictions, i)
            tn, fp, fn, tp = metrics.confusion_matrix(
                speech_frame_ground_truth,
                speech_frame_prob_predictions > i).ravel()
            sub_fer = 100 * ((fp + fn) / len(speech_frame_ground_truth))
            logger.info(
                f"PFa {100*mp_fa:.2f} Pmiss {100*mp_miss:.2f} FER {sub_fer:.2f} t: {i:.2f}"
            )

        auc = metrics.roc(speech_frame_ground_truth,
                          speech_frame_prob_predictions) * 100
        for avgtype in ('micro', 'macro', 'binary'):
            precision, recall, f1, _ = metrics.precision_recall_fscore_support(
                speech_frame_ground_truth,
                speech_frame_predictions,
                average=avgtype)
            vad_results.append(
                (avgtype, 100 * precision, 100 * recall, 100 * f1))

        logger.info("Calculating segment based metric .. ")
        # Change order just for better printing in file
        prediction_df = prediction_df[[
            'filename', 'onset', 'offset', 'event_label'
        ]]
        metric = metrics.segment_based_evaluation_df(
            label_df, prediction_df, time_resolution=time_resolution)
        logger.info("Calculating event based metric .. ")
        event_metric = metrics.event_based_evaluation_df(
            label_df, prediction_df)

        prediction_df.to_csv(experiment_path /
                             f'speech_predictions_{task}.tsv',
                             sep='\t',
                             index=False)
        full_prediction_df.to_csv(experiment_path / f'predictions_{task}.tsv',
                                  sep='\t',
                                  index=False)
        with open(experiment_path / f'evaluation_{task}.txt', 'w') as fp:
            for k, v in config_parameters.items():
                print(f"{k}:{v}", file=fp)
            print(metric, file=fp)
            print(event_metric, file=fp)
            for avgtype, precision, recall, f1 in vad_results:
                print(
                    f"VAD {avgtype} F1: {f1:<10.3f} {precision:<10.3f} Recall: {recall:<10.3f}",
                    file=fp)
            print(f"FER: {fer:.2f}", file=fp)
            print(f"AUC: {auc:.2f}", file=fp)
            print(f"Pfa: {p_fa:.2f}", file=fp)
            print(f"Pmiss: {p_miss:.2f}", file=fp)
            print(f"ACC: {acc:.2f}", file=fp)
        logger.info(f"Results are at {experiment_path}")
        for avgtype, precision, recall, f1 in vad_results:
            print(
                f"VAD {avgtype:<10} F1: {f1:<10.3f} Pre: {precision:<10.3f} Recall: {recall:<10.3f}"
            )
        print(f"FER: {fer:.2f}")
        print(f"AUC: {auc:.2f}")
        print(f"Pfa: {p_fa:.2f}")
        print(f"Pmiss: {p_miss:.2f}")
        print(f"ACC: {acc:.2f}")
        print(event_metric)
        print(metric)
Пример #17
0
print("XvalidT.shape: {}".format(XvalidT.shape))
print("YvalidT.shape: {}".format(YvalidT.shape))
# (2) Define model
nfeature = Xtrain.shape[0]
model = tf.keras.models.Sequential([
 tf.keras.layers.Dense(200,input_shape=(nfeature,),activation="tanh"),
 tf.keras.layers.Dense(50,activation="tanh"),
 tf.keras.layers.Dense(1,activation="sigmoid")])
# (3) Compile model
optimizer = tf.keras.optimizers.Adam(lr=0.02,beta_1=0.9,beta_2=0.999,epsilon=1e-7)
model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=["accuracy"])
model.summary()
# (4) Train model
epochs = 20
time_start = time.time()
ntrain = XtrainT.shape[0]
history = model.fit(XtrainT,YtrainT,epochs=epochs,batch_size=ntrain,validation_data=(XvalidT,YvalidT))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Predictions and plotting
# confusion matrix
Afinal = model.predict(XvalidT).T
Yvalid_pred = np.round(Afinal)
metrics.confusion_matrix(Yvalid,Yvalid_pred,2)
f1score,precision,recall = metrics.f1score(Yvalid,Yvalid_pred)
print("F1Score: {} - Precision: {} - Recall: {}".format(f1score,precision,recall))
text_results.text_results(Yvalid,Yvalid_pred,Xvalid_raw)
# plot loss and accuracy
plot_results.plot_results_history(history.history,["loss","val_loss"])
plot_results.plot_results_history(history.history,["accuracy","val_accuracy"])
plt.show()
Пример #18
0
 def evaluate(self, actual: np.array, predicted: np.array) -> confusion_matrix:
     return confusion_matrix(actual, predicted, labels=[0,1])
Пример #19
0
            train_X, test_X = data.iloc[train_index], data.iloc[test_index]
            train_y, test_y = y.iloc[train_index], y.iloc[test_index]

            clf = None
            clf = RandomForestClassifier(n_estimators=p,
                                         criterion='entropy',
                                         max_depth=14,
                                         min_samples_split=20,
                                         n_jobs=2)

            clf.fit(train_X, train_y)
            results.append(metrics.predict_table(clf, test_X, test_y))

        result = pd.concat(results)

        matrix = metrics.confusion_matrix(result)

        clases = matrix.columns.tolist()
        precisions = [metrics.precision(matrix, c) for c in clases]
        recalls = [metrics.recall(matrix, c) for c in clases]
        f_scores = [metrics.f_score(matrix, c) for c in clases]

        w_score = metrics.weighted_f_score(matrix)

        # f = open(result_dir + str(max_depth) + ' ' + str(min_samples_split) + '.txt', 'w')
        f = open(result_dir + str(p) + '.txt', 'w')

        f.write('F_score by class')
        f.write('\n')
        f.write(str(f_scores))
        f.write('\n')
Пример #20
0
def valid(model, validation_loader, loss, best_loss, pad_size, orig_dim,
          num_img):
    # set the model to be ready to test on the validation set
    model.eval()

    validation_acc = 0.0
    validation_loss = 0.0
    validation_IoU = 0.0
    IoU_count = 0.0

    # test the model
    print("Testing model...", flush=True)

    # run through the batches of validation images
    for validation_imgs, validation_labels in validation_loader:
        # use GPU for images if possible
        if torch.cuda.is_available():
            validation_imgs = validation_imgs.cuda()
            validation_labels = validation_labels.cuda()

        # predict the classes (whether a pixel contains a building or not)
        outputs = model(validation_imgs)

        # compute the loss
        loss_output = loss(outputs, validation_labels.long())

        # calculate the validation loss and find the prediction for each pixel of an image
        validation_loss += loss_output.item() * validation_imgs.size(0)
        _, prediction = torch.max(outputs.data, 1)

        # if the image and label were padded beforehand crop them back into their original
        # dimensions
        if pad_size != 0:
            prediction = crop(prediction.float(), orig_dim)
            validation_labels = crop(validation_labels, orig_dim)

        for img in range(validation_labels.shape[0]):
            # set up the confusion matrix and calculate the metrics
            # check that there is an object to compute the metrics with
            if torch.sum(validation_labels[img]) != 0:
                IoU_count += 1
                _, _, IoU = confusion_matrix(
                    np.asarray(prediction[img].cpu()),
                    np.asarray(validation_labels[img].cpu()), orig_dim)
                validation_IoU += IoU

        # calculate the accuracy after each batch
        validation_acc += torch.sum(
            prediction.long() == validation_labels.long().data)

    # calculate the total accuracy, validation loss, and IoU
    validation_acc = validation_acc.item() / (float(num_img) * orig_dim *
                                              orig_dim)
    validation_loss = validation_loss / num_img

    # make sure there is no divide by 0
    if IoU_count != 0:
        validation_IoU = validation_IoU / IoU_count

    print("Val acc: " + str(validation_acc),
          "Val loss: " + str(validation_loss),
          flush=True)
    print("Val IoU: " + str(validation_IoU), flush=True)

    # save the model's learned parameters with the lowest loss
    if best_loss > validation_loss:
        best_loss = validation_loss
        save_model(model)

    return best_loss
Пример #21
0
    feature_filter = args.feature_filter

    train_data = pd.read_csv(training_set_path, index_col=0)
    train_X, train_y = utils.filter_data(train_data, feature_filter=feature_filter)

    test_data = pd.read_csv(test_set_path, index_col=0)
    test_X, test_y = utils.filter_data(test_data, feature_filter=feature_filter)


    clf = None
    clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
                                 max_depth=max_depth, min_samples_split=min_samples_split,
                                 n_jobs=n_processes)

    clf.fit(train_X, train_y)
    result = metrics.predict_table(clf, test_X, test_y)

    result['indice'] = test_X.index.tolist()
    result.set_index('indice')
    result.index.name = catalog + '_id'
    result = result.drop('indice', axis=1)

    output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+')
    pickle.dump(clf, output)
    output.close()

    result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv')

    print metrics.weighted_f_score(metrics.confusion_matrix(result))
Пример #22
0
# bikin confusion matrix dan metric dari training ini:
prediction = model.predict(X_test)
label_pred = []
for i in range(prediction.shape[1]):
    label_pred.append(np.argmax(prediction[:, i]))
y_test_label = []
for i in range(y_test.shape[0]):
    y_test_label.append(np.argmax(y_test[i, :]))
metrics = Metrics(y_test_label, label_pred)
print("ACCURACY SLURRR: ", metrics.all_accuracy())
print("PRECISION SLURRR: ", metrics.all_precision())
print("RECALL SLURRR: ", metrics.all_recall())
print("F1 SLURRR: ", metrics.all_f1_score())
print("CONFUSION MATRIXX: ")
print(confusion_matrix(y_test_label, label_pred))

print("--------------------------------------")

# 10-fold cross validation

k_fold = KFold(n_splits=10)
print(k_fold)
scores = []
for train_index, test_index in k_fold.split(X):
    print("TRAIN DATA INDEX")
    print(train_index)
    print("TEST DATA INDEX")
    print(test_index)
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
Пример #23
0
    results = []
    skf = cross_validation.StratifiedKFold(y, n_folds=folds)
    for train_index, test_index in skf:
        train_X, test_X = data.iloc[train_index], data.iloc[test_index]
        train_y, test_y = y.iloc[train_index], y.iloc[test_index]

        clf = None
        clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion,
                                     max_depth=max_depth, min_samples_split=min_samples_split,
                                     n_jobs=n_processes)

        clf.fit(train_X, train_y)
        results.append(metrics.predict_table(clf, test_X, test_y))

    result = pd.concat(results)

    output = open(result_path + 'Arboles/Arbol.pkl', 'wb+')
    pickle.dump(clf, output)
    output.close()

    result.to_csv(result_path + 'Predicciones/result.csv')

    matrix = metrics.confusion_matrix(result)
    matrix.to_csv(result_path + 'Metricas/soft_matrix_.csv')

    clases = matrix.columns.tolist()
    f_score = [metrics.f_score(matrix, c) for c in clases]

    with open(result_path + 'Metricas/results.txt') as f:
        f.write(clases + '\n')
        f.write(str(f_score) + '\n')
Пример #24
0
    def validate(self, dataloader, is_val_set=True, measure_entropy=True):
        self.net.eval()
        self.hook.flag_hook = True

        prefix = self.get_prefix(is_val_set)
        self.metrics_epoch = collections.defaultdict(utils.Meter)
        matrix = np.zeros((self.c_dim, self.c_dim), dtype=np.uint32)
        for mb, (x, y) in enumerate(dataloader):
            x, y = x.to(self.device), y.to(self.device)
            y_one_hot = utils.to_one_hot(y, self.c_dim)

            logits = self.net(x)
            losses = self.criterion(logits, y_one_hot)

            matrix = matrix + metrics.confusion_matrix(
                utils.tensor2array(utils.get_class_outputs(logits)),
                utils.tensor2array(y), self.c_dim)
            self.metrics_epoch['{}_loss'.format(prefix)].update(
                utils.tensor2array(losses), x.shape[0])

            if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
                num_log = min(self.cfg.num_log, x.shape[0])
                name = '{}_{}_{}_epoch{:0=3d}_minibatch{}'
                filepath = '{}/{}'.format(
                    os.path.join(self.cfg.plot_dir, self.cfg.model_type,
                                 self.cfg.model_name), name)
                x_ = x[0:num_log]
                x_np, y_np = utils.tensor2array(
                    x[0:num_log]), utils.tensor2array(y[0:num_log])
                losses_np = utils.tensor2array(losses[0:num_log])

                plotting.make_grid(
                    x_,
                    filepath.format(prefix, 'data', 'x',
                                    self.metrics['epochs'][-1], mb + 1))
                utils.save_array(
                    x_np,
                    filepath.format(prefix, 'data', 'x',
                                    self.metrics['epochs'][-1], mb + 1))
                utils.save_array(
                    y_np,
                    filepath.format(prefix, 'data', 'y',
                                    self.metrics['epochs'][-1], mb + 1))
                utils.save_array(
                    losses_np,
                    filepath.format(prefix, 'data', 'losses',
                                    self.metrics['epochs'][-1], mb + 1))

                for (k, layer_name) in enumerate(self.hook.layers):
                    layer_np = utils.tensor2array(
                        self.hook.layers[layer_name][0:num_log])
                    utils.save_array(
                        layer_np,
                        filepath.format(prefix, 'data', layer_name,
                                        self.metrics['epochs'][-1], mb + 1))

            if measure_entropy:
                entropy = metrics.entropy(utils.logits_to_probs(logits))
                self.metrics_epoch['{}_entropy'.format(prefix)].update(
                    utils.tensor2array(entropy), x.shape[0])

                if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
                    entropy_np = utils.tensor2array(entropy[0:num_log])
                    utils.save_array(
                        entropy_np,
                        filepath.format(prefix, 'data', 'entropy',
                                        self.metrics['epochs'][-1], mb + 1))

                if is_val_set:
                    x_rand = torch.randn(size=x.shape).to(self.device)
                    logits_rand = self.net(x_rand)
                    entropy_rand = metrics.entropy(
                        utils.logits_to_probs(logits_rand))
                    self.metrics_epoch['entropy_rand'].update(
                        utils.tensor2array(entropy_rand), x.shape[0])

                    if self.cfg.num_log > 0 and self.cfg.plot and mb == 0:
                        name = '{}_{}_{}_epoch{:0=3d}_minibatch{}'
                        filepath = '{}/{}'.format(
                            os.path.join(self.cfg.plot_dir,
                                         self.cfg.model_type,
                                         self.cfg.model_name), name)
                        x_ = x_rand[0:num_log]
                        x_np = utils.tensor2array(x_rand[0:num_log])
                        entropy_np = utils.tensor2array(
                            entropy_rand[0:num_log])

                        plotting.make_grid(
                            x_,
                            filepath.format(prefix, 'noise', 'x',
                                            self.metrics['epochs'][-1],
                                            mb + 1))
                        utils.save_array(
                            x_np,
                            filepath.format(prefix, 'noise', 'x',
                                            self.metrics['epochs'][-1],
                                            mb + 1))
                        utils.save_array(
                            entropy_np,
                            filepath.format(prefix, 'noise', 'entropy',
                                            self.metrics['epochs'][-1],
                                            mb + 1))

                        for (k, layer_name) in enumerate(self.hook.layers):
                            layer_np = utils.tensor2array(
                                self.hook.layers[layer_name][0:num_log])
                            utils.save_array(
                                layer_np,
                                filepath.format(prefix, 'noise', layer_name,
                                                self.metrics['epochs'][-1],
                                                mb + 1))

            # disable hook after first minibatch by default - this is done for computational/speed purposes
            self.hook.flag_hook = False

        self.summarize_metrics(matrix, prefix)