def test_classifier_deprecated(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
scikit_learn.KerasClassifier(build_fn_clf)
assert len(w) == 1
assert issubclass(w[-1].category, DeprecationWarning)
assert 'KerasClassifier is deprecated' in str(w[-1].message)
def mlp_tuning(X_train, X_test, Y_train, Y_test):
model = wp.KerasClassifier(build_fn=multi_layer_perceptron,
input_dim=X_train.shape[1])
grid_params = {
'activation': ['relu', 'sigmoid', 'tanh'],
'num_nodes': np.arange(10, 120, 10)
}
cv = GridSearchCV(estimator=model,
param_grid=grid_params,
n_jobs=-1,
verbose=2,
cv=5)
grid_result = cv.fit(X_train, Y_train)
print("Best: %f using %s", (grid_result.best_params_))
numnodes = grid_result.best_params_['num_nodes']
activation = grid_result.best_params_['activation']
soft_values, predictions, training_soft_values, training_predictions, accuracy, fmeasure, macro_gmean, training_accuracy, training_fmeasure, training_macro_gmean = train_test(
X_train,
Y_train,
X_test,
Y_test,
num_nodes=numnodes,
activation=activation)
print(accuracy)
return accuracy
def model_selection(x, y, look_back):
# define the grid search parameters
batch_size = [16, 64, 128]
epochs = [10, 25, 50]
layers = [{
'input': 64,
'output': 1
}, {
'input': 128,
'output': 1
}, {
'input': 256,
'output': 1
}, {
'input': 128,
'hidden1': 64,
'output': 1
}]
dropout_rate = [0.2, 0.5, 0.8]
# batch_size = np.random.choice(batch_size, int(len(batch_size) / 2))
# epochs = np.random.choice(epochs, int(len(epochs) / 2))
# layers = np.random.choice(layers, int(len(layers) / 2))
# learning_rate = np.random.choice(learning_rate, int(len(learning_rate) / 2))
# dropout_rate = np.random.choice(dropout_rate, int(len(dropout_rate) / 2))
model = scikit_learn.KerasClassifier(build_fn=create_model,
look_back=look_back,
num_features=len(x[0, 0, :]),
verbose=0)
param_grid = dict(layers=layers,
batch_size=batch_size,
epochs=epochs,
dropout_rate=dropout_rate)
'''
validation_fold = [-1 for _ in range(len(x_train))] + [0 for _ in range(len(x_val))]
x_train = np.append(x_train, x_val, axis=0)
y_train = np.append(y_train, y_val, axis=0)
ps = PredefinedSplit(validation_fold)
grid = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_iter=100, n_jobs=-1, cv=ps, scoring="roc_auc", verbose=1)
'''
# grid = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_iter=50, n_jobs=-1, cv=3, scoring="roc_auc", verbose=0)
grid = GridSearchCV(estimator=model,
param_grid=param_grid,
n_jobs=-1,
cv=3,
scoring="roc_auc",
verbose=3)
grid_result = grid.fit(x, y)
print("Best: %f using %s" %
(grid_result.best_score_, grid_result.best_params_))
params = grid_result.best_params_
return create_fit_model(x_train,
y_train,
look_back,
params=params,
times_to_repeat=1)
def test_classify_build_fn(self):
with self.cached_session():
clf = scikit_learn.KerasClassifier(build_fn=build_fn_clf,
hidden_dim=HIDDEN_DIM,
batch_size=BATCH_SIZE,
epochs=EPOCHS)
assert_classification_works(clf)
def train_model(df):
""" Train multiple models with different params
Args:
df (`DataFrame`): DataFrame used to train the model
"""
data = df.copy()
logging.config.fileConfig(config.LOGGING_CONFIG)
logger = logging.getLogger('model_training')
logger.info("The shape of the data is {} ".format(data.shape))
# feature engineering
X_train, X_test, y_train, y_test = FE.train_test_split_stratified(
data) # train test split with stratified sampling
X_train_vec, X_test_vec = FE.FE_text_feature(
X_train, X_test) # extract text feature
X_train_SD, X_test_SD = FE.FE_standardize(X_train_vec,
X_test_vec) # standardization
X_train_OH, X_test_OH = FE.FE_onehot(X_train_SD,
X_test_SD) # one-hot coding
# transform a keras model to a scikit learn model object
model = scikit_learn.KerasClassifier(build_fn=create_model,
input_dim=X_train_OH.shape[1],
verbose=0)
# get params space from model_config.yml file
with open(
os.path.join(config.PROJECT_HOME, 'model', 'src',
'model_config.yml'), 'r') as f:
params_training = yaml.load(f)
batch_size = params_training["model_tuning"]["batch_size"]
epochs = params_training["model_tuning"]["epochs"]
optimizer = params_training["model_tuning"]["optimizer"]
scoring = params_training["model_tuning"]["scoring"]
# grid search to find the best hyperparams combination
param_grid = dict(optimizer=optimizer,
batch_size=batch_size,
epochs=epochs)
grid = GridSearchCV(estimator=model,
param_grid=param_grid,
scoring=scoring,
n_jobs=1,
cv=3)
grid_result = grid.fit(X_train_OH, y_train)
# print the result of the model
logger.info('Best: {} using {}'.format(grid_result.best_score_,
grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, std, param in zip(means, stds, params):
logger.info("%f (%f) with: %r" % (mean, std, param))
def test_classify_class_build_fn(self):
class ClassBuildFnClf(object):
def __call__(self, hidden_dim):
return build_fn_clf(hidden_dim)
with self.cached_session():
clf = scikit_learn.KerasClassifier(build_fn=ClassBuildFnClf(),
hidden_dim=HIDDEN_DIM,
batch_size=BATCH_SIZE,
epochs=EPOCHS)
assert_classification_works(clf)
def search_param():
model = scikit_learn.KerasClassifier(build_fn=FM,
epochs=50,
batch_size=32,
verbose=0)
k = range(10, 200, 10)
param_grid = dict(k=k)
grid = GridSearchCV(
estimator=model, param_grid=param_grid, n_jobs=2
) #, scoring = make_scorer(f1_score, greater_is_better = False))
X_train, X_test, y_train, y_test = vectorization_main()
grid_result = grid.fit(X_train, y_train)
print('Best: {} using {}'.format(grid_result.best_score_,
grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, std, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, std, param))
def cnn_model():
# read data
X_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')
# set parameter and preprocess data
batchSize = 100
targetNum = 10
epoch = 10
X_train = X_train.reshape(len(X_train), 28, 28, 1)
X_test = X_test.reshape(len(X_test), 28, 28, 1)
input_shape = [28, 28, 1]
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train /= 255
X_test /= 255
y_train = keras.utils.to_categorical(y_train, targetNum)
y_test = keras.utils.to_categorical(y_test, targetNum)
# make a dic of hyperparameter and choose best one
parameter = {'learning_rate': [0.001, 0.01, 0.1]}
model1 = scikit_learn.KerasClassifier(build_fn=creat_conv_model)
grid1 = GridSearchCV(estimator=model1, param_grid=parameter)
grid_result1 = grid1.fit(X_train, y_train)
best_lr = grid_result1.best_params_['learning_rate']
# use best model to train
best_model = creat_conv_model(best_lr)
history = best_model.fit(X_train,
y_train,
batch_size=batchSize,
epochs=epoch,
validation_split=0.2,
shuffle=True)
# print result of grid search
print('-' * 30)
print('validation accuracy: {}, best parameter: {}'.format(
grid_result1.best_score_, grid_result1.best_params_['learning_rate']))
means = grid_result1.cv_results_['mean_test_score']
params = grid_result1.cv_results_['params']
for mean, param in zip(means, params):
print('{} with {}'.format(mean, param))
print('-' * 30)
# print test accuracy
loss, accuracy = best_model.evaluate(X_test, y_test)
print('test set accuracy: {}'.format(accuracy))
print('-' * 30)
# make confusion matrix
y_pre = best_model.predict(X_test)
y_pred = np.argmax(y_pre, axis=1)
y_test = np.argmax(y_test, axis=1)
print('confusion matrix')
print(confusion_matrix(y_test, y_pred))
print('-' * 30)
# save model
save_dir = './cnn_model'
model_name = 'cnn.h5'
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
model_path = os.path.join(save_dir, model_name)
best_model.save(model_path)
print('save trained model at %s' % model_path)
# draw training loss graph
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'valid'], loc='upper left')
plt.savefig('cnn_loss.png')
plt.show()
filename = generate_dataset_filename(dataset_identifier=dataset_identifier)
dataset = open_dataset_file(filename, preprocessed_datasets_folder)
(X_train, y_train, ground_truth_train, t0_train), \
(X_val, y_val, ground_truth_val, t0_val), \
(X_test, y_test, ground_truth_test, t0_test), \
scaling_factor = generate_training_data_lstm(dataset, train_cv_test_split=train_cv_test_split, cleanse=False)
# 设置种子，为了可复现（这个无关紧要）
seed = 7
np.random.seed(seed)
model = scikit_learn.KerasClassifier(build_fn=generate_model,
verbose=1,
epochs=100)
batch_size = [64, 128]
#epochs = [50, 100, 200]
dropout_rate = [0.5, 0.7]
learning_rate = [0.01, 0.1]
nb_hidden_layers = [1, 2]
nb_hidden_neurons = [40, 50]
param_grid = dict(batch_size=batch_size,
dropout_rate=dropout_rate,
learning_rate=learning_rate,
nb_hidden_layers=nb_hidden_layers,
nb_hidden_neurons=nb_hidden_neurons)
grid = GridSearchCV(estimator=model,
def modeldump(df, sav_path=None):
""" Train the best model and save to file
Args:
df (`DataFrame`): DataFrame used to train the momdel
sav_path (`str`): Path to save the model. Defaults to None
"""
np.random.seed(423)
tf.random.set_seed(423)
os.environ['PYTHONHASHSEED'] = str(423)
session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(),
config=session_conf)
tf.compat.v1.keras.backend.set_session(sess)
data = df.copy()
# get params space from model_config.yml file
with open(
os.path.join(config.PROJECT_HOME, 'model', 'src',
'model_config.yml'), 'r') as f:
params_training = yaml.load(f)
batch_size = params_training["model_production"]["batch_size"]
epochs = params_training["model_production"]["epochs"]
optimizer = params_training["model_production"]["optimizer"]
X_train, X_test, y_train, y_test = FE.train_test_split_stratified(
data) # train test split with stratified sampling
X_train_vec, X_test_vec = FE.FE_text_feature(
X_train, X_test) # extract text feature
X_train_SD, X_test_SD = FE.FE_standardize(X_train_vec,
X_test_vec) # standardization
X_train_OH, X_test_OH = FE.FE_onehot(X_train_SD,
X_test_SD) # one-hot coding
g = tf.Graph()
with g.as_default():
tf.random.set_seed(423)
model = scikit_learn.KerasClassifier(build_fn=MT.create_model,
input_dim=X_train_OH.shape[1],
verbose=0,
optimizer=optimizer,
batch_size=batch_size,
epochs=epochs)
model.fit(X_train_OH, y_train)
y_pred = model.predict(X_test_OH)
cm = confusion_matrix(y_test, y_pred)
with open(sav_path + 'result.txt', 'w') as f:
f.write("Size of input: {0}\n".format(X_test_OH.shape[1]))
f.write("The confusion matrix on the test data is as follows:\n")
f.write(str(cm))
f.write("\n")
#print("Size of input:", X_test_OH.shape)
#print("The confusion matrix on the test data is as follows:")
#print(cm)
y = data['fraudulent'].copy()
X = data.drop(['fraudulent'], axis=1)
X = FE.FE_text_feature(
X, sav_path=(config.MODEL_PATH +
'vec_file.pickle')) # extract text feature
X = FE.FE_standardize(X, sav_path=(config.MODEL_PATH +
'SD_file.pickle')) # standardization
X = FE.FE_onehot(X, sav_path=(config.MODEL_PATH +
'OH_file.pickle')) # one-hot coding
g = tf.Graph()
with g.as_default():
tf.random.set_seed(423)
model = scikit_learn.KerasClassifier(build_fn=MT.create_model,
input_dim=X.shape[1],
verbose=0,
optimizer=optimizer,
batch_size=batch_size,
epochs=epochs)
model.fit(X, y)
y_pred = model.predict(X)
cm = confusion_matrix(y, y_pred)
with open(sav_path + 'result.txt', 'a') as f:
f.write("Size of input: {0}\n".format(X.shape[1]))
f.write(
"The confusion matrix on the training data is as follows:\n")
f.write(str(cm))
f.write("\n")
model.model.save_weights(sav_path + 'model_file.ckpt')
with open(sav_path + 'input_size.txt', 'w') as f:
f.write(str(X.shape[1]))
Dense(units=7,
kernel_initializer='RandomNormal',
activation='sigmoid',
input_dim=9))
model.add(
Dense(units=6, kernel_initializer='RandomNormal', activation='relu'))
model.add(
Dense(units=1, kernel_initializer='RandomNormal',
activation='sigmoid'))
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
sk_model = scikit_learn.KerasClassifier(build_fn=create_model)
epochs = [100, 120, 150, 170, 200]
batches = [3, 5, 7, 9]
param_grid = dict(epochs=epochs, batch_size=batches)
grid = model_selection.GridSearchCV(estimator=sk_model,
param_grid=param_grid,
n_jobs=-1)
grid_result = grid.fit(X_train, y_train)
# summarize results
print("Best: %f using %s" %
(grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
def keras_wrappers():
keras_classifier = scikit_learn.KerasClassifier()
keras_classifier.predict()
def multi_layer_model():
# read data
X_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')
# process data
batchSize = 100
epoch = 10
targetNum = 10
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train /= 255
X_test /= 255
y_train = keras.utils.to_categorical(y_train, targetNum)
y_test = keras.utils.to_categorical(y_test, targetNum)
# make a dict of hyperparameter and chose the best
parameter1 = {'learning_rate': [0.001,0.01,0.1],
'hidden_nodes': [100]}
model1 = scikit_learn.KerasClassifier(build_fn=creat_model)
grid1 = GridSearchCV(estimator=model1,param_grid=parameter1)
grid_result1 = grid1.fit(X_train,y_train)
best_lr = grid_result1.best_params_['learning_rate']
parameter2 = {'learning_rate': [best_lr],
'hidden_nodes': [100,200,300]}
model2 = scikit_learn.KerasClassifier(build_fn=creat_model)
grid2 = GridSearchCV(estimator=model2,param_grid=parameter2)
grid_result2 = grid2.fit(X_train,y_train)
best_nodes = grid_result2.best_params_['hidden_nodes']
# use best model to train again
best_model = creat_model(best_lr,best_nodes)
history = best_model.fit(X_train,y_train,
batch_size=batchSize,
epochs=epoch,
validation_split=0.2,
shuffle=True)
# print result of grid search
print('-'*30)
print('accurary: {}, best parameter: {}'.format(grid_result1.best_score_, grid_result1.best_params_['learning_rate']))
means = grid_result1.cv_results_['mean_test_score']
params = grid_result1.cv_results_['params']
for mean, param in zip(means, params):
print("%f with: %r" % (mean,param))
print('-'*30)
print('accurary: {}, best parameter: {}'.format(grid_result2.best_score_, grid_result2.best_params_['hidden_nodes']))
means = grid_result2.cv_results_['mean_test_score']
params = grid_result2.cv_results_['params']
for mean, param in zip(means, params):
print("%f with: %r" % (mean, param))
print('-'*30)
# print test accuracy
loss,accuracy = best_model.evaluate(X_test,y_test)
print('test set accuracy: {}'.format(accuracy))
print('-'*30)
# make confusion matrix
y_pre = best_model.predict(X_test)
y_pred = np.argmax(y_pre,axis=1)
y_test = np.argmax(y_test,axis=1)
print('confusion matrix')
print(confusion_matrix(y_test,y_pred))
# save model
save_dir = './dnn_model'
model_name = 'dnn.h5'
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
model_path = os.path.join(save_dir,model_name)
best_model.save(model_path)
print('save trained model at %s' % model_path)
plt.figure()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train','valid'],loc='upper left')
plt.savefig('dnn_loss.png')
plt.show()
# load training dataset
training_dataset = np.loadtxt(fname_training, delimiter=",")
# split into input (X) and output (Y) variables
X = training_dataset[:,0:feature_number]
Y = training_dataset[:,feature_number] #last col of the training set is the target
# specify general CV settings
folds = 5
# specify general NN CV settings
n_epochs = 30
n_batch_size = 50
# evaluate model with the original dataset
estimator = keras_scikit.KerasClassifier(build_fn=create_baseline, nb_epoch=n_epochs,
batch_size=n_batch_size, verbose=0)
kfold = msel.StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)
results = msel.cross_val_score(estimator, X, Y, cv=kfold)
print("Evaluation Results (baseline model, raw dataset: mean score, std): %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))
# evaluate baseline model with the standardized (normally scaled) dataset
np.random.seed(seed)
estimators = []
estimators.append(('standardize', preproc.StandardScaler()))
estimators.append(('mlp', keras_scikit.KerasClassifier(build_fn=create_baseline, epochs=n_epochs,
batch_size=n_batch_size, verbose=0)))
pipeline = pipe.Pipeline(estimators)
kfold = msel.StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)
results = msel.cross_val_score(pipeline, X, Y, cv=kfold)
print("Evaluation Results (baseline model, standardized dataset: mean score, std): %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))
