def train_models(): if len(utils.CATEGORIES) == 0 or utils.not_enough_images(): err = utils.get_not_enough_images_error() return jsonify(error=err) utils.delete_all_models() utils.set_maybe_old(True) maybe_update_models() try: regression_train.train() cnn_train.train() except: err = "Unknown error." utils.delete_all_models() return jsonify(error=err) if utils.train_should_stop(): utils.delete_all_models() utils.train_should_stop(False) else: utils.set_maybe_old(False) utils.reset_progress() return "ok"
def experiment(options): dataset_name = options['dataset_name'] urep_class = options['urep_class'] urep_ratio = options['urep_ratio'] use_validation_step = options['use_validation_step'] train_size = options['train_size'] test_size = options['test_size'] n_splits = options['n_splits'] X, y = datasets.load_data(dataset_name) y = y.astype(int) n_classes = y.max() + 1 if urep_class is not None: test_data_imbalance(y, urep_class) eval = [ 0, np.zeros(n_classes), np.zeros(n_classes), 0, np.zeros(n_classes), np.zeros(n_classes) ] if use_validation_step: sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, train_size=train_size) train_index, test_index = next(sss.split(X, y)) val_index = np.delete(np.arange(X.shape[0]), np.concatenate((train_index, test_index))) X_train, y_train = shuffle(X[train_index], y[train_index]) X_val, y_val = shuffle(X[val_index], y[val_index]) X_test, y_test = shuffle(X[test_index], y[test_index]) if urep_class is not None and urep_ratio is not None: X_train, y_train = force_data_imbalance(X_train, y_train, urep_class, urep_ratio) test_data_imbalance(y_train, urep_class) eval = train(X_train, y_train, X_val, y_val, X_test, y_test, options) else: skf = StratifiedKFold(n_splits=n_splits) for train_index, test_index in skf.split(X, y): X_train, y_train = shuffle(X[train_index], y[train_index]) X_test, y_test = shuffle(X[test_index], y[test_index]) if urep_class is not None and urep_ratio is not None: X_train, y_train = force_data_imbalance( X_train, y_train, urep_class, urep_ratio) test_data_imbalance(y_train, urep_class) split_eval = train(X_train, y_train, None, None, X_test, y_test, options) for i in range(6): eval[i] += split_eval[i] / n_splits for perf in eval: print(perf) return eval
def train(self, train_X, train_Y): self.word_vec_array = make_array_of_vecs(self.word_to_index, self.train_word_vecs, self.params, train=True) train_X, self.params['MAX_LENGTH'] = to_dense(train_X) train_Y = one_hot(train_Y, self.params['CLASSES']) if self.hp['flex']: self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH']) else: self.params['FLEX'] = 0 self.best_epoch_path, self.word_vec_array = cnn_train.train(self.params, train_X, train_Y, self.word_vec_array, self.model_dir)
def train(self, train_X, train_Y): self.vocab = get_vocab(self.indices_to_words) self.word_vec_array = make_array_of_vecs(self.vocab, self.word_vecs, self.params, train=True) train_X, self.params['MAX_LENGTH'] = to_dense(train_X) train_Y = one_hot(train_Y, self.params['CLASSES']) if self.hp['flex']: self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH']) else: self.params['FLEX'] = 0 self.best_epoch_path, self.word_vec_array = cnn_train.train(self.params, train_X, train_Y, self.word_vec_array, self.model_dir)
def train(self, train_X, train_Y): self.vocab = get_vocab(self.indices_to_words) self.key_array = dict_to_array(self.word2vec_filename, self.vocab, self.params) train_X, self.params['MAX_LENGTH'] = to_dense(train_X) train_Y = one_hot(train_Y, self.params['CLASSES']) # train_X = collapse_vectors(train_X, params['WORD_VECTOR_LENGTH']) if self.hp['flex']: self.params['FLEX'] = int(self.hp['flex_amt'] * self.params['MAX_LENGTH']) else: self.params['FLEX'] = 0 self.best_epoch_path, self.key_array = cnn_train.train(self.params, train_X, train_Y, self.key_array, self.model_dir)
def single_run(config, training=True): start = time.time() if config['model_fn'].endswith('hdf5'): # using preset config model = models.use_saved_model(config['model_fn'], **config) else: model_fn = getattr(models, config['model_fn']) model = model_fn(**config) model.summary() print('Model compiled after {}'.format(runtime(start))) tmp = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S_%f") config['tmp'] = tmp config_name = '../config/config_{}_{}.json'.format(tmp, config['name']) output_name = '../output/out_{}_{}.json'.format(tmp, config['name']) config['model_w_name'] = "../weights/weights_{}_{}.hdf5".format( config['name'], tmp) config['output_name'] = output_name print('Saving configuration file to: {}'.format(config_name), flush=True) print('and output file to: {}'.format(output_name), flush=True) with open(config_name, 'w') as f: json.dump(config, f, indent=4) print('Using following configuration:') print(json.dumps(config, indent=2)) if training: labels, data, meta = create_dataset('train.json', True, **config) if config.get('pseudo_train', False): idxs, test, test_meta = create_dataset('test.json', False, **config) dataset = ((labels, data, meta), (idxs, test, test_meta)) else: dataset = (labels, data, meta) print('Data loaded after {}'.format(runtime(start))) model = train(dataset, model, **config) print('Model trained after {}'.format(runtime(start))) idxs, test, test_meta = create_dataset('test.json', False, **config) dataset = (idxs, test, test_meta) evaluate(model, dataset, **config) print('Scriped successfully completed after {}'.format(runtime(start)))
(X, y_orig) = image2array(input_dir_prefix, nb_classes, img_tensor_shape) # setup X X = X.astype('float32') X /= 255 # convert class vectors to binary class matrices Y = np_utils.to_categorical(y_orig, nb_classes) sampleNum = X.shape[0] trainSampleNum = math.floor(sampleNum * trainSampleRatio) print('sampleNum:', sampleNum) randomIndices = np.random.permutation(list(range(sampleNum))) X = X[randomIndices] Y = Y[randomIndices] print('X.shape: ', X.shape) X_train = X[:trainSampleNum] X_test = X[trainSampleNum + 1:sampleNum] Y_train = Y[:trainSampleNum] Y_test = Y[trainSampleNum + 1:] print('X_train shape:', X_train.shape) print(X_train.shape[0], 'train samples') print('Y_train shape:', Y_train.shape) print('X_test shape:', X_test.shape) print(X_test.shape[0], 'test samples') print('Y_test shape:', Y_test.shape) train(X_train, Y_train, X_train, Y_train, img_colors, img_rows, img_cols, batch_size, nb_epoch, data_augmentation) predict(X_test, Y_test)
# -*- coding: utf-8 -*- """ Created on Mon Nov 13 19:18:42 2017 @author: HM-PC 数据:训练集为55000张图片;验证集为5000张图片;测试集为10000张图片;每张图片大小为28x28; 每张图片精处理为784的一维数组(28x28);像素矩阵的值为[0,1]之间,0表示白色背景,1表示黑色前景 """ import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import numpy as np import cnn_train import cnn_test import LeNet5 import AlexNet if __name__ == '__main__': mnist = input_data.read_data_sets("F:/datasets/MNIST_data", one_hot=True) cnn_train.train(AlexNet, mnist, "AlexNet") #cnn_test.evaluate(LeNet5, mnist, "AlexNet")