Beispiel #1
0
def confusion_mt(model, test_x, test_y):
    y_pred = model.predict(test_x)
    y_pred = np.argmax(y_pred, axis=1)
    cm = sk_metrics.confusion_matrix(y_true=test_y, y_pred=y_pred)
    plt.figure()
    plot_confusion_matrix(cm, hide_ticks=True,cmap=plt.cm.Blues)
    plt.show()
Beispiel #2
0
def plot_confusion(model, X, y):
    """
    Plot confusion matrix of model classifications on data.
    :param model: model to make predictions on images
    :param X: features
    :param y: labels
    """
    predictions = model.predict(X).argmax(axis=-1)
    predictions = predictions.reshape(1, -1)[0]

    correct = np.nonzero(predictions == y)[0]
    incorrect = np.nonzero(predictions != y)[0]
    print("Correct:", len(correct))
    print("Incorrect:", len(incorrect))

    y_pred = model.predict(X)
    # to get the prediction, we pick the class with with the highest probability
    y_pred_classes = np.argmax(y_pred, axis=1)
    y_true = np.argmax(to_categorical(y, len(LABELS)), axis=1)

    conf_mtx = confusion_matrix(y_true, y_pred_classes)
    plot_confusion_matrix(conf_mtx, figsize=(12, 8), hide_ticks=True, cmap=plt.cm.Blues)
    plt.xticks(range(3), LABELS, fontsize=16)
    plt.yticks(range(3), LABELS, fontsize=16)
    plt.show()
Beispiel #3
0
def index():
    project_id = 'weighty-flag-288919'
    bucket_name = 'delabs2020'
    client = storage.Client(project=project_id)
    bucket = client.get_bucket(bucket_name)
    blob = bucket.blob('results/predictions-00000-of-00001.csv')
    blob.download_to_filename('download.csv')

    # Load necessary data
    results = pd.read_csv('download.csv',
                          header=None,
                          names=['predicted', 'actual'])

    # Generate the figure **without using pyplot**.
    cm = confusion_matrix(results['actual'], results['predicted'])
    plt.figure()
    plot_confusion_matrix(cm,
                          figsize=(7.5, 5),
                          hide_ticks=True,
                          cmap=plt.cm.Blues)
    plt.title("K Neighbors Model - Confusion Matrix")
    plt.xticks(range(2), ["Heart Not Failed", "Heart Fail"], fontsize=16)
    plt.yticks(range(2), ["Heart Not Failed", "Heart Fail"],
               fontsize=16,
               rotation=45)
    # Save it to a temporary buffer.
    buf = BytesIO()
    plt.savefig(buf, format="png")

    # Embed the result in the html output.
    data = base64.b64encode(buf.getbuffer()).decode("ascii")
    return render_template('index.html', data=data)
Beispiel #4
0
def test_model(API_KEY, df_test):
    y_true = df_test.genre
    y_pred = []

    for i, row in df_test.reset_index().iterrows():
        demo = classifyText(API_KEY, row['lyric'])
        y_pred.append(demo["class_name"])

    cm = confusion_matrix(y_true,
                          y_pred,
                          labels=["hip_hop", "funk", "sertanejo"])
    plot_confusion_matrix(conf_mat=cm,
                          show_normed=True,
                          figsize=(5, 5),
                          class_names=["hip_hop", "funk", "sertanejo"])
    plt.tight_layout()
    plt.savefig('assets/confusion_matrix.png')

    report = classification_report(
        y_true,
        y_pred,
        target_names=["hip_hop", "funk", "sertanejo"],
        output_dict=True)
    metrics = pd.DataFrame(report).transpose()
    metrics.to_csv('assets/metrics.csv')
Beispiel #5
0
def train(ctx, vocab_size, num_classes, filter_num, batch_size,
          word_embed_size, training_steps, learning_rate, print_loss_every,
          confusion_matrix, keep_proba, filter_sizes, save_model):

    # Load dataset
    (x_train, y_train), (x_test, y_test) = get_dataset(ctx.train_path,
                                                       ctx.test_path)
    sequence_length = x_train.shape[1]
    dataset_size = x_train.shape[0]

    tf.reset_default_graph()
    with tf.Graph().as_default():
        cnn = TextCNN(sequence_length, vocab_size, word_embed_size,
                      filter_sizes, filter_num, num_classes)

        # Set eval feed_dict
        train_feed_dict = {
            cnn.input_x: x_train,
            cnn.input_y: y_train,
            cnn.keep_proba: 1.0
        }
        test_feed_dict = {
            cnn.input_x: x_test,
            cnn.input_y: y_test,
            cnn.keep_proba: 1.0
        }

        # Train
        saver = tf.train.Saver()
        train_step = tf.train.AdamOptimizer(learning_rate).minimize(cnn.loss)
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(training_steps):
                start = (i * batch_size) % dataset_size
                end = min(start + batch_size, dataset_size)
                feed_dict = {
                    cnn.input_x: x_train[start:end],
                    cnn.input_y: y_train[start:end],
                    cnn.keep_proba: keep_proba
                }
                sess.run(train_step, feed_dict=feed_dict)
                if i % print_loss_every == 0:
                    avg_cost = cnn.loss.eval(feed_dict=feed_dict)
                    train_acc = cnn.accuracy.eval(feed_dict=train_feed_dict)
                    test_acc = cnn.accuracy.eval(feed_dict=test_feed_dict)
                    test_pred = cnn.pred.eval(feed_dict=test_feed_dict)
                    print(f"Epoch: {i:04d} | AvgCost: {avg_cost:7.4f}", end="")
                    print(f" | Train/Test ACC: {train_acc:.3f}/{test_acc:.3f}")

            # After training, save the sess
            if save_model:
                save_path = saver.save(sess, SESS_PATH)
                print('Model state has been saved!')

        if confusion_matrix:
            binary = cm(y_true=y_test, y_pred=test_pred)
            print('\n', 'Confusion Matrix: ')
            print(binary)
            plot_confusion_matrix(binary)
            plt.show()
Beispiel #6
0
 def evaluate_g(self, test_x, test_y):
     y_pre = self.combined.predict(test_x)
     y_pre = np.argmax(y_pre, axis=1)
     cm = metrics.confusion_matrix(y_true=test_y, y_pred=y_pre)  # shape=(12, 12)
     plt.figure()
     plot_confusion_matrix(cm, hide_ticks=True,cmap=plt.cm.Blues,figsize=(8,8))
     plt.show()
Beispiel #7
0
def run_svm(df):
    X = df['text']
    y = df['polarity']

    tfidf = TfidfVectorizer(norm = 'l1', ngram_range=(1,2), analyzer='word', max_features=5000)

    X = tfidf.fit_transform(X)
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 0, stratify = y)

    print('shape of X:', X.shape)
    print('')
    
    clf = LogisticRegression(penalty= 'l2', C= 2.7825594022071245, solver = 'liblinear', max_iter= 100)

    clf.fit(X_train, y_train)

    y_pred = clf.predict(X_test)

    print("Printing Report")

    print(classification_report(y_test, y_pred))
    
    # accuracy plot
    plot_confusion_matrix(confusion_matrix(y_test, y_pred))
    plt.show()
    
    return tfidf, clf
def main():
    print('Stage 0: Importing Packages')
    param = getParameters()
    data, labels, dataOriginal = getData(param['data_path'], param['s'])
    trainData, testData, trainLabels, testLabels = splitData(
        data, labels, param['train_images_indices'],
        param['test_images_indices'])
    model = createModel(s=param['s'],
                        optimizer=param['optimizer'],
                        outputNeurons=param['classes'],
                        lr=param['learnRate'],
                        dropoutProp=param['dropRate'],
                        layers=param['layers'],
                        flat=param['flat'],
                        max=param['max'],
                        decay=param['decay'])
    model, rrr = trainModel(model=model,
                            data=trainData,
                            labels=trainLabels,
                            aug=param['aug'],
                            num=['numOfAug'],
                            s=param['s'],
                            val=param['valRate'],
                            batch=param['batch'],
                            epochs=param['epochs'])
    acc, predicts, confusion = prediction(model=model,
                                          data=testData,
                                          labels=testLabels,
                                          threshold=param['threshold'])
    print("The accuracy rate is:" + str(acc))
    plot_confusion_matrix(conf_mat=confusion)
    plt.suptitle('Confusion-Matrix')
    plotPrecisionRecall(testLabels, predicts)
    return
Beispiel #9
0
def run_test_harness():
	# load dataset
    trainX, trainY, testX, testY = load_dataset()
	# prepare pixel data
    trainX, testX = prep_pixels(trainX, testX)
	# define model
    model = define_model()

    # DATA AUGMENTATION:
    # making more copies of the dataset with small modifications
    # create data generator
    datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True)
	# prepare iterator
    it_train = datagen.flow(trainX, trainY, batch_size=64)
	# fit model
    steps = int(trainX.shape[0] / 64)

	# fit model
    history = model.fit_generator(it_train, steps_per_epoch=steps, epochs=200, validation_data=(testX, testY), verbose=1)

	# evaluate model
    _, acc = model.evaluate(testX, testY, verbose=0)
    print('> %.3f' % (acc * 100.0))

    # learning curves
    plot_diagnostics(history)
    classes_name = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']

    #plot confusion matrix
    predY = model.predict_classes(testX)
    rounded_labels = np.argmax(testY, axis = 1)
    mat = confusion_matrix(rounded_labels, predY)
    plot_confusion_matrix(mat, figsize=(9,9), show_normed = True, class_names = classes_name)

    model.save('CIFAR10_2DENSELAYER_MODEL.h5')
def create_test_model():
    # Example of testing bact vs viral
    test_model = create_empty_model(True)
    test_model.load_weights('best_bactviral_checkpoint.hdf5')

    loss, acc = test_model.evaluate(bactviral_test_data,
                                    bactviral_test_labels,
                                    verbose=2)
    print('Restored model, accuracy: {:5.2f}%'.format(100 * acc))

    # Get predictions
    preds = test_model.predict(bactviral_test_data, batch_size=16)
    preds = np.argmax(preds, axis=-1)

    # Original labels
    orig_test_labels = np.argmax(bactviral_test_labels, axis=-1)

    cm = confusion_matrix(orig_test_labels, preds)
    plt.figure()
    plot_confusion_matrix(cm,
                          figsize=(12, 8),
                          hide_ticks=True,
                          cmap=plt.cm.Blues)
    plt.xticks(range(2), ['Bacterial', 'Viral'], fontsize=16)
    plt.yticks(range(2), ['Bacterial', 'Viral'], fontsize=16)
    plt.show()
Beispiel #11
0
def evaluate_model(name='vgg16', filename=None):
    if filename is None:
        filename = 'data/' + name + 'cm.png'
    filepath = filepath_for(name)
    model, _, _ = build_model(name, filepath)

    test_data, test_labels = load_test_data()
    preds = model.predict(test_data, batch_size=config.BATCH_SIZE)
    preds = np.argmax(preds, axis=-1)

    orig_test_labels = np.argmax(test_labels, axis=-1)
    cm  = confusion_matrix(orig_test_labels, preds)
    plot_confusion_matrix(cm,figsize=(12,8), hide_ticks=True, cmap=plt.cm.BuPu)
    plt.xticks(range(2), ['Normal', 'Pneumonia'], fontsize=16)
    plt.yticks(range(2), ['Normal', 'Pneumonia'], fontsize=16)
    plt.savefig(filename)

    tn, fp, fn, tp = cm.ravel()

    precision = tp/(tp+fp)
    recall = tp/(tp+fn)

    print("Recall of the model is {:.2f}".format(recall))
    print("Precision of the model is {:.2f}".format(precision))
    print("f1 score is {:.2f}".format(2 * (recall * precision)/(recall + precision)))
    del model
    del test_data
    del test_labels
    del preds
    del cm  
Beispiel #12
0
def calc_cm(Y_true,
            Y_pred,
            output_file=None,
            labels=None,
            figsize=(8.5, 7.5),
            colorbar=True,
            show_absolute=True,
            show_normed=True,
            ylim=None):
    cm = confusion_matrix(Y_true, Y_pred)
    acc = np.trace(cm) / np.sum(cm).astype('float')
    loss = 1 - acc
    if output_file:
        plot_confusion_matrix(conf_mat=cm,
                              colorbar=colorbar,
                              show_absolute=show_absolute,
                              show_normed=show_normed,
                              class_names=labels,
                              figsize=figsize)
        plt.title("Confusion Matrix\nAccuracy={:0.4f}; Loss={:0.4f}".format(
            acc, loss))
        plt.tight_layout()
        if ylim:
            plt.ylim(ylim)
        plt.savefig(output_file)
    return (cm, acc, loss)
Beispiel #13
0
def executeKnn(X_train, X_test, y_train, y_test):
    from sklearn.neighbors import KNeighborsClassifier
    classifier = KNeighborsClassifier(n_neighbors=5)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    plot_confusion_matrix(y_test, y_pred)
    from sklearn.metrics import accuracy_score
    acuracia = accuracy_score(y_test, y_pred)
    print('Acurácia KNN: ', acuracia)
Beispiel #14
0
def executeDecisionTree(X_train, X_test, y_train, y_test):
    from sklearn.tree import DecisionTreeClassifier
    classifier = DecisionTreeClassifier()
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    plot_confusion_matrix(y_test, y_pred)
    from sklearn.metrics import accuracy_score
    acuracia = accuracy_score(y_test, y_pred)
    print('Acurácia DecisionTree: ', acuracia)
def mlp_classifier(train_x, train_y, test_x, test_y):

    classifier = MLPClassifier(hidden_layer_sizes=(100, 100))
    classifier.fit(train_x, train_y)
    prediction = classifier.predict(test_x)
    accuracy = metrics.accuracy_score(test_y, prediction)
    print("Accuracy for MLP classifier:", accuracy)
    plot_confusion_matrix(test_y, prediction, "MLP")
    return classifier
def svm_classifier(train_x, train_y, test_x, test_y):

    classifier = svm.SVC(kernel="linear")
    classifier.fit(train_x, train_y)
    prediction = classifier.predict(test_x)
    accuracy = metrics.accuracy_score(test_y, prediction)
    print("Accuracy for the SVM classifier:", accuracy)
    plot_confusion_matrix(test_y, prediction, "SVM")
    return classifier
Beispiel #17
0
def executeSVM(X_train, X_test, y_train, y_test):
    from sklearn.svm import SVC
    classifier = SVC(gamma='auto', kernel='rbf')  # Kernel Linear
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    plot_confusion_matrix(y_test, y_pred)
    from sklearn.metrics import accuracy_score
    acuracia = accuracy_score(y_test, y_pred)
    print('Acurácia SVM: ', acuracia)
Beispiel #18
0
def executeRandomForest(X_train, X_test, y_train, y_test):
    from sklearn.ensemble import RandomForestClassifier
    classifier = RandomForestClassifier()
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    plot_confusion_matrix(y_test, y_pred)
    from sklearn.metrics import accuracy_score
    acuracia = accuracy_score(y_test, y_pred)
    print('Acurácia RandomForest: ', acuracia)
Beispiel #19
0
def executeMPL(X_train, X_test, y_train, y_test):
    from sklearn.neural_network import MLPClassifier
    classifier = MLPClassifier(alpha=1e-5,
                               hidden_layer_sizes=(5, 5),
                               random_state=1)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    plot_confusion_matrix(y_test, y_pred)
    from sklearn.metrics import accuracy_score
    acuracia = accuracy_score(y_test, y_pred)
    print('Acurácia MPL: ', acuracia)
def generate_confusion_matrix(predictions, y_validation):
    cm = confusion_matrix(y_validation, predictions)
    plt.figure()
    plt.style.use("dark_background")
    plot_confusion_matrix(cm,
                          figsize=(12, 8),
                          hide_ticks=True,
                          cmap=plt.cm.Greens)
    plt.xticks(range(2), ['Normal', 'Pneumonia'], fontsize=16)
    plt.yticks(range(2), ['Normal', 'Pneumonia'], fontsize=16)
    plt.show()
Beispiel #21
0
def evaluate_model_metric(embbeder, supports, x_test, y_test ,k_shot=1, metric='l2'):
    x_test_3 = triple_channels(x_test)

    y_pred = classify_by_metric(embbeder, supports,
                              x_test_3, k_shot=k_shot,
                              metric=metric)

    cm = sk_metrics.confusion_matrix(y_true=y_test, y_pred=y_pred)
    plt.figure()
    plot_confusion_matrix(cm, hide_ticks=True,cmap=plt.cm.Blues)
    plt.show()
    return (y_pred == y_test).mean(), 0
Beispiel #22
0
    def confusion_matrix(self):
        """ Prints/displays the confusion matrix. """

        self.matrix = confusion_matrix(self.data.y_test.argmax(axis=1),
                                       self.test_preds.argmax(axis=1))

        self.helpers.logger.info("Confusion Matrix: " + str(self.matrix))
        print("")

        plot_confusion_matrix(conf_mat=self.matrix)
        plt.savefig('model/plots/confusion-matrix.png')
        plt.show()
        plt.clf()
Beispiel #23
0
def train(ctx, vocab_size, num_classes, filter_num, batch_size,
          word_embed_size, training_steps, learning_rate, print_loss_every,
          confusion_matrix):

    # Load dataset
    train = np.loadtxt(ctx.train_path, dtype=int)
    test = np.loadtxt(ctx.test_path, dtype=int)
    x_train = train[:, :-1]
    y_train = train[:, -1:].reshape((-1, ))
    x_test = test[:, :-1]
    y_test = test[:, -1:].reshape((-1, ))
    sequence_length = x_train.shape[1]

    with tf.Graph().as_default():
        cnn = TextCNN(sequence_length, vocab_size, word_embed_size,
                      num_classes, filter_num)

        # Set feed_dict
        input_x, input_y = cnn.input_x, cnn.input_y
        train_feed_dict = {input_x: x_train, input_y: y_train}
        test_feed_dict = {input_x: x_test, input_y: y_test}

        # Train
        train_step = tf.train.AdamOptimizer(learning_rate).minimize(cnn.loss)
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(training_steps):
                batch_data = train[
                    np.random.randint(train.shape[0], size=batch_size), :]
                X = batch_data[:, :-1]
                Y = batch_data[:, -1:].reshape((-1, ))
                feed_dict = {input_x: X, input_y: Y}
                sess.run(train_step, feed_dict=feed_dict)
                if i % print_loss_every == 0:
                    total_cross_entropy = cnn.loss.eval(feed_dict=feed_dict)
                    train_accuracy = cnn.accuracy.eval(
                        feed_dict=train_feed_dict)
                    test_accuracy = cnn.accuracy.eval(feed_dict=test_feed_dict)
                    test_pred = cnn.pred.eval(feed_dict=test_feed_dict)
                    print(
                        "After %d training steps, cross entropy on batch data is"
                        " %f, trian accuracy is %.2f, test accuracy is %.2f" %
                        (i, total_cross_entropy, train_accuracy,
                         test_accuracy))

        if confusion_matrix:
            binary = cm(y_true=y_test, y_pred=test_pred)
            print('\n', 'Confusion Matrix: ')
            print(binary)
            plot_confusion_matrix(binary)
            plt.show()
Beispiel #24
0
def executeSVM(iris_ds, df_iris, X_train, X_test, y_train, y_test):
    """
    Algoritimo supervisionado de classificação por SVM.
    Hiperplano de separação das classes.
    """
    # Treinamento do modelo
    from sklearn.svm import SVC
    classifier = SVC(gamma='auto')  # Kernel Linear
    classifier.fit(X_train, y_train)

    # Previsão
    y_pred = classifier.predict(X_test)

    plot_confusion_matrix(y_test, y_pred)
Beispiel #25
0
def plot(model):
    test_X = np.load('dataset/processed/test_x.npy')
    test_Y = np.load('dataset/processed/test_y.npy')

    Y_pred = model.predict(np.array(test_X))
    y_pred = np.around(Y_pred)

    cm = multilabel_confusion_matrix(test_Y, y_pred)
    plt.figure()

    plot_confusion_matrix(cm[0], figsize=(12, 8), hide_ticks=True)
    plt.xticks(range(2), ['SPAM', 'HAM'], fontsize=16)
    plt.yticks(range(2), ['SPAM', 'HAM'], fontsize=16)
    plt.savefig("matrix.png")
    plt.show()
Beispiel #26
0
def executeKnn(iris_ds, df_iris, X_train, X_test, y_train, y_test):
    """
    Algoritimo supervisionado que determina o rótulo de classificação de uma 
    amostra baseado nas amostras vizinhas advindas de um conjunto de treinamento
    """

    # Treinamento do modeloiris_ds, df_iris, X_train, X_test, y_train, y_test
    from sklearn.neighbors import KNeighborsClassifier
    classifier = KNeighborsClassifier(n_neighbors=5)
    classifier.fit(X_train, y_train)

    # Previsão
    y_pred = classifier.predict(X_test)

    plot_confusion_matrix(y_test, y_pred)
Beispiel #27
0
def executeDecisionTree(iris_ds, df_iris, X_train, X_test, y_train, y_test):
    """
    Algoritimo supervisionado de classificação por árvore de decisão.
    """
    from sklearn.tree import DecisionTreeClassifier

    classifier = DecisionTreeClassifier()
    classifier.fit(X_train, y_train)

    # Previsão
    y_pred = classifier.predict(X_test)

    plot_confusion_matrix(y_test, y_pred)
    plot_arvore_decisao(classifier, iris_ds['feature_names'])
    plot_arvore_decisao
Beispiel #28
0
def plot_knn(data, grid, y_true, y_pred):
    X_plot = data['X_train'].values
    y_plot = data['y_train'].values.astype(np.integer)
    plot_decision_regions(X_plot, y_plot, clf=grid.best_estimator_, legend=2, scatter_kwargs=dict(s=20), markers='+o')

    cm = confusion_matrix(y_true, y_pred)
    fig, ax = plot_confusion_matrix(conf_mat=cm, show_absolute=True, show_normed=True, colorbar=True)
def display_confusion_matrix(grid, input_train, target_train, filedir=None, taskname=None):
    """
    Display prediction in confusion matrix
    
    Parameters
    ----------
    grid: Object
        GridSearchCV object
    input_train_fs: 2D-list
    target_train: 2D-list
    filedir: str, Optional, default=None
        Directory for file
    taskname: str, Optional, default=None
        Name of file
    datapoints: str, Optional, default=350
        How many datapoints that should be displayed in plot
    """
    X_train, X_test, y_train, y_test = train_test_split(input_train, target_train, test_size=0.33, random_state=42)
    
    model, predict = predict_model(grid, X_train, y_train, X_test)
    
    cm = confusion_matrix(y_test,predict)
    
    fig, ax = plot_confusion_matrix(conf_mat=cm,
                                show_absolute=True,
                                show_normed=True,
                                colorbar=True)
    ax.set_title("Confusion matrix - "  + taskname)
    if taskname and filedir: plt.savefig(filedir +"Pictures/"+ taskname + "_prediction.png", format='png')
    plt.show()
Beispiel #30
0
def plot_accuracy(method, cm, accuracy):
    print("Classifier: " + method)
    print("Confusion matrix: \n")
    print(cm)
    print("Accuracy: " + str(accuracy) + "\n")
    fig, ax = plot_confusion_matrix(conf_mat=cm,
                                    show_absolute=False,
                                    colorbar=True,
                                    show_normed=True)
    plt.title("Confusion matrix " + method)
    x = [
        "LUAD", "BRCA", "KIRC", "COAD", "OV", "READ", "LUSC", "GBM", "UCEC",
        "HNSC"
    ]
    y = [
        "LUAD", "BRCA", "KIRC", "COAD", "OV", "READ", "LUSC", "GBM", "UCEC",
        "HNSC"
    ]
    lx = list(range(10))
    ly = list(range(10))
    ax.set_xticks(lx)
    ax.set_xticklabels(x)
    ax.set_yticks(ly)
    ax.set_yticklabels(y)
    plt.show()