def train_models():
if len(utils.CATEGORIES) == 0 or utils.not_enough_images():
err = utils.get_not_enough_images_error()
return jsonify(error=err)
utils.delete_all_models()
utils.set_maybe_old(True)
maybe_update_models()
try:
regression_train.train()
cnn_train.train()
except:
err = "Unknown error."
utils.delete_all_models()
return jsonify(error=err)
if utils.train_should_stop():
utils.delete_all_models()
utils.train_should_stop(False)
else:
utils.set_maybe_old(False)
utils.reset_progress()
return "ok"
def experiment(options):
dataset_name = options['dataset_name']
urep_class = options['urep_class']
urep_ratio = options['urep_ratio']
use_validation_step = options['use_validation_step']
train_size = options['train_size']
test_size = options['test_size']
n_splits = options['n_splits']
X, y = datasets.load_data(dataset_name)
y = y.astype(int)
n_classes = y.max() + 1
if urep_class is not None:
test_data_imbalance(y, urep_class)
eval = [
0,
np.zeros(n_classes),
np.zeros(n_classes), 0,
np.zeros(n_classes),
np.zeros(n_classes)
]
if use_validation_step:
sss = StratifiedShuffleSplit(n_splits=1,
test_size=test_size,
train_size=train_size)
train_index, test_index = next(sss.split(X, y))
val_index = np.delete(np.arange(X.shape[0]),
np.concatenate((train_index, test_index)))
X_train, y_train = shuffle(X[train_index], y[train_index])
X_val, y_val = shuffle(X[val_index], y[val_index])
X_test, y_test = shuffle(X[test_index], y[test_index])
if urep_class is not None and urep_ratio is not None:
X_train, y_train = force_data_imbalance(X_train, y_train,
urep_class, urep_ratio)
test_data_imbalance(y_train, urep_class)
eval = train(X_train, y_train, X_val, y_val, X_test, y_test, options)
else:
skf = StratifiedKFold(n_splits=n_splits)
for train_index, test_index in skf.split(X, y):
X_train, y_train = shuffle(X[train_index], y[train_index])
X_test, y_test = shuffle(X[test_index], y[test_index])
if urep_class is not None and urep_ratio is not None:
X_train, y_train = force_data_imbalance(
X_train, y_train, urep_class, urep_ratio)
test_data_imbalance(y_train, urep_class)
split_eval = train(X_train, y_train, None, None, X_test, y_test,
options)
for i in range(6):
eval[i] += split_eval[i] / n_splits
for perf in eval:
print(perf)
return eval
def train(self, train_X, train_Y):
self.word_vec_array = make_array_of_vecs(self.word_to_index, self.train_word_vecs,
self.params, train=True)
train_X, self.params['MAX_LENGTH'] = to_dense(train_X)
train_Y = one_hot(train_Y, self.params['CLASSES'])
if self.hp['flex']:
self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH'])
else:
self.params['FLEX'] = 0
self.best_epoch_path, self.word_vec_array = cnn_train.train(self.params,
train_X, train_Y, self.word_vec_array, self.model_dir)
def train(self, train_X, train_Y):
self.vocab = get_vocab(self.indices_to_words)
self.word_vec_array = make_array_of_vecs(self.vocab, self.word_vecs, self.params, train=True)
train_X, self.params['MAX_LENGTH'] = to_dense(train_X)
train_Y = one_hot(train_Y, self.params['CLASSES'])
if self.hp['flex']:
self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH'])
else:
self.params['FLEX'] = 0
self.best_epoch_path, self.word_vec_array = cnn_train.train(self.params,
train_X, train_Y, self.word_vec_array, self.model_dir)
def train(self, train_X, train_Y):
self.vocab = get_vocab(self.indices_to_words)
self.key_array = dict_to_array(self.word2vec_filename, self.vocab, self.params)
train_X, self.params['MAX_LENGTH'] = to_dense(train_X)
train_Y = one_hot(train_Y, self.params['CLASSES'])
# train_X = collapse_vectors(train_X, params['WORD_VECTOR_LENGTH'])
if self.hp['flex']:
self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH'])
else:
self.params['FLEX'] = 0
self.best_epoch_path, self.key_array = cnn_train.train(self.params,
train_X, train_Y, self.key_array, self.model_dir)
def single_run(config, training=True):
start = time.time()
if config['model_fn'].endswith('hdf5'):
# using preset config
model = models.use_saved_model(config['model_fn'], **config)
else:
model_fn = getattr(models, config['model_fn'])
model = model_fn(**config)
model.summary()
print('Model compiled after {}'.format(runtime(start)))
tmp = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S_%f")
config['tmp'] = tmp
config_name = '../config/config_{}_{}.json'.format(tmp, config['name'])
output_name = '../output/out_{}_{}.json'.format(tmp, config['name'])
config['model_w_name'] = "../weights/weights_{}_{}.hdf5".format(
config['name'], tmp)
config['output_name'] = output_name
print('Saving configuration file to: {}'.format(config_name), flush=True)
print('and output file to: {}'.format(output_name), flush=True)
with open(config_name, 'w') as f:
json.dump(config, f, indent=4)
print('Using following configuration:')
print(json.dumps(config, indent=2))
if training:
labels, data, meta = create_dataset('train.json', True, **config)
if config.get('pseudo_train', False):
idxs, test, test_meta = create_dataset('test.json', False,
**config)
dataset = ((labels, data, meta), (idxs, test, test_meta))
else:
dataset = (labels, data, meta)
print('Data loaded after {}'.format(runtime(start)))
model = train(dataset, model, **config)
print('Model trained after {}'.format(runtime(start)))
idxs, test, test_meta = create_dataset('test.json', False, **config)
dataset = (idxs, test, test_meta)
evaluate(model, dataset, **config)
print('Scriped successfully completed after {}'.format(runtime(start)))
(X, y_orig) = image2array(input_dir_prefix, nb_classes, img_tensor_shape)
# setup X
X = X.astype('float32')
X /= 255
# convert class vectors to binary class matrices
Y = np_utils.to_categorical(y_orig, nb_classes)
sampleNum = X.shape[0]
trainSampleNum = math.floor(sampleNum * trainSampleRatio)
print('sampleNum:', sampleNum)
randomIndices = np.random.permutation(list(range(sampleNum)))
X = X[randomIndices]
Y = Y[randomIndices]
print('X.shape: ', X.shape)
X_train = X[:trainSampleNum]
X_test = X[trainSampleNum + 1:sampleNum]
Y_train = Y[:trainSampleNum]
Y_test = Y[trainSampleNum + 1:]
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print('Y_train shape:', Y_train.shape)
print('X_test shape:', X_test.shape)
print(X_test.shape[0], 'test samples')
print('Y_test shape:', Y_test.shape)
train(X_train, Y_train, X_train, Y_train, img_colors, img_rows, img_cols,
batch_size, nb_epoch, data_augmentation)
predict(X_test, Y_test)
# -*- coding: utf-8 -*-
"""
Created on Mon Nov 13 19:18:42 2017
@author: HM-PC
数据:训练集为55000张图片；验证集为5000张图片；测试集为10000张图片；每张图片大小为28x28；
每张图片精处理为784的一维数组（28x28）；像素矩阵的值为[0,1]之间，0表示白色背景，1表示黑色前景
"""
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np
import cnn_train
import cnn_test
import LeNet5
import AlexNet
if __name__ == '__main__':
mnist = input_data.read_data_sets("F:/datasets/MNIST_data", one_hot=True)
cnn_train.train(AlexNet, mnist, "AlexNet")
#cnn_test.evaluate(LeNet5, mnist, "AlexNet")