Python read_dataset Examples

Example #1

File: train.py Project: idea7766/ML2017FALL

from keras.models import Sequential

import read_n_write as rw
import utils
import models

base_dir = os.path.dirname(__file__)
DATA_PATH = sys.argv[1]
LOG_FOLDER = './model_log'
CHECK_POINT_NAME = LOG_FOLDER + '/model.{epoch:04d}-{val_acc:.4f}.h5'
LOG_NAME = LOG_FOLDER + '/log.csv'

PATIENCE = 50
EPOCH = 2000

feats, lables, _ = rw.read_dataset(DATA_PATH, shuffle=True)
X_train, Y_train, X_val, Y_val = utils.validation(feats, lables, 0.05)

train_data_gen = ImageDataGenerator(rotation_range=40,
                                    width_shift_range=0.2,
                                    height_shift_range=0.2,
                                    shear_range=0.2,
                                    zoom_range=0.2,
                                    horizontal_flip=True,
                                    fill_mode='nearest')
train_data_gen.fit(X_train)

val_data_gen = ImageDataGenerator()

callback = [
    TensorBoard(),

Example #2

from keras.models import load_model, Sequential
from keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
import itertools
import numpy as np
import matplotlib.pyplot as plt

import utils
import read_n_write as rw

MODEL_PATH = './model_re_bst.h5'
DATA_PATH = './data/train.csv'

model = load_model(MODEL_PATH)
x, y, _ = rw.read_dataset(DATA_PATH)
y = utils.to_nb_class(y)

y_pred = model.predict_classes(x)
# y_pred = to_categorical(y_pred)
# y_pred = utils.to_nb_class(y_pred)
print(y_pred)

cnf_mat = confusion_matrix(y, y_pred)


def plot_confusion_matrix(cm,
                          classes,
                          title='Confusion matrix',
                          cmap=plt.cm.jet):
    """
    This function prints and plots the confusion matrix.

Example #3

import os
import sys
from keras.models import Sequential, load_model

import utils
import read_n_write as rw

TEST_PATH = sys.argv[1]
ANS_PATH = sys.argv[2]
MODELS_PATH = ['./model_re_bst.h5',
               './model_ta.h5',
               './model_x.h5']
WEIGHT = [1.06, 1, 1]
x_test = rw.read_dataset('data/test.csv', labeled_data = False)

model_ls = []
result_ls = []

num_model = len(MODELS_PATH)

for i, w, mod_path in zip(range(num_model), WEIGHT, MODELS_PATH):
# io processing
    model  = load_model(mod_path)
    model_ls.append(model)
    result = model.predict(x_test)
    result_ls.append(result)
    if i == 0:
        result_total = w * result
    else:
        result_total += w * result
    # print(result_total)

Example #4

import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, CSVLogger, ModelCheckpoint, TensorBoard, ReduceLROnPlateau
from keras.models import Sequential

import read_n_write as rw
import utils
import models

base_dir = os.path.dirname(__file__)
TRAIN_FOLDER = './model_chk/dnn1'
CHECK_POINT_NAME = TRAIN_FOLDER + '/model.{epoch:04d}-{val_acc:.4f}.h5'
LOG_NAME = TRAIN_FOLDER + '/log.csv'
PATIENCE = 50

feats, lables, _ = rw.read_dataset('data/train.csv', shuffle=True)
X_train, Y_train, X_val, Y_val = utils.validation(feats, lables, 0.1)

train_data_gen = ImageDataGenerator(rotation_range=40,
                                    width_shift_range=0.2,
                                    height_shift_range=0.2,
                                    shear_range=0.2,
                                    zoom_range=0.2,
                                    horizontal_flip=True,
                                    fill_mode='nearest')
train_data_gen.fit(X_train)

val_data_gen = ImageDataGenerator()

# early_stopping = EarlyStopping(patience = 50)
callback = [

Example #5

File: predict.py Project: idea7766/ML2017FALL

import os
import sys
from keras.models import Sequential, load_model

import utils
import read_n_write as rw

base_dir = os.path.dirname(__file__)

MODEL_PATH = './model_re_bst.h5'
TEST_PATH = sys.argv[1]
SAVE_PATH = sys.argv[2]

# io processing
model = load_model(MODEL_PATH)
x = rw.read_dataset(TEST_PATH, labeled_data=False)

y_pred = model.predict(x)
y_pred = utils.to_nb_class(y_pred)
rw.save_dir(y_pred, path=SAVE_PATH)