def test_scaler_without_centering(): rng = np.random.RandomState(42) X = rng.randn(4, 5) X[:, 0] = 0.0 # first feature is always of zero scaler = Scaler(with_mean=False) X_scaled = scaler.fit(X).transform(X, copy=True) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal( X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert X_scaled is not X X_scaled_back = scaler.inverse_transform(X_scaled) assert X_scaled_back is not X assert X_scaled_back is not X_scaled assert_array_almost_equal(X_scaled_back, X) X_scaled = scale(X, with_mean=False) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal( X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert X_scaled is not X X_scaled_back = scaler.inverse_transform(X_scaled) assert X_scaled_back is not X assert X_scaled_back is not X_scaled assert_array_almost_equal(X_scaled_back, X)
def test_scaler_1d(): """Test scaling of dataset along single axis""" rng = np.random.RandomState(0) X = rng.randn(5) X_orig_copy = X.copy() scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert_array_almost_equal(X_scaled_back, X_orig_copy) # Test with 1D list X = [0., 1., 2, 0.4, 1.] scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) X_scaled = scale(X) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
def test_scaler_1d(): """Test scaling of dataset along single axis""" rng = np.random.RandomState(0) X = rng.randn(5) X_orig_copy = X.copy() scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert_array_almost_equal(X_scaled_back, X_orig_copy) # Test with 1D list X = [0., 1., 2, 0.4, 1.] scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) X_scaled = scale(X) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
def SVM_fit(X_in, y_in, X_out, gamma, C): M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breakes the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1 * len(X_in)))) shuffle(X_in, y_in) X_test = [X_in[i] for i in test_indices] y_test = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_test, y_test) X_in = scaler.transform(X_in) X_test = scaler.transform(X_test) X_out = scaler.transform( X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_test.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_test = [[X_test[i][j] for j in f_indices] for i in range(len(X_test))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(f_indices) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered svc = svm.SVC(kernel='rbf', C=C, gamma=gamma, verbose=False, cache_size=4092, tol=1e-5) svc.fit(X_in, y_in) y_out = svc.predict(X_out) return y_out
def test_scaler_2d_arrays(): """Test scaling of 2d array along first axis""" rng = np.random.RandomState(0) X = rng.randn(4, 5) X[:, 0] = 0.0 # first feature is always of zero scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is not X) # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert_true(X_scaled_back is not X) assert_true(X_scaled_back is not X_scaled) assert_array_almost_equal(X_scaled_back, X) X_scaled = scale(X, axis=1, with_std=False) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) X_scaled = scale(X, axis=1, with_std=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0]) # Check that the data hasn't been modified assert_true(X_scaled is not X) X_scaled = scaler.fit(X).transform(X, copy=False) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is X) X = rng.randn(4, 5) X[:, 0] = 1.0 # first feature is a constant, non zero feature scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is not X)
def test_scaler_2d_arrays(): """Test scaling of 2d array along first axis""" rng = np.random.RandomState(0) X = rng.randn(4, 5) X[:, 0] = 0.0 # first feature is always of zero scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is not X) # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert_true(X_scaled_back is not X) assert_true(X_scaled_back is not X_scaled) assert_array_almost_equal(X_scaled_back, X) X_scaled = scale(X, axis=1, with_std=False) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) X_scaled = scale(X, axis=1, with_std=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0]) # Check that the data hasn't been modified assert_true(X_scaled is not X) X_scaled = scaler.fit(X).transform(X, copy=False) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is X) X = rng.randn(4, 5) X[:, 0] = 1.0 # first feature is a constant, non zero feature scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert_false(np.any(np.isnan(X_scaled))) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert_true(X_scaled is not X)
def SVM_fit(X_in, y_in, X_out, gamma, C): M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breakes the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1*len(X_in)))) shuffle(X_in, y_in) X_test = [X_in[i] for i in test_indices] y_test = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_test, y_test) X_in = scaler.transform(X_in) X_test = scaler.transform(X_test) X_out = scaler.transform(X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_test.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_test = [[X_test[i][j] for j in f_indices] for i in range(len(X_test))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(f_indices) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered svc = svm.SVC(kernel='rbf', C=C, gamma=gamma, verbose=False, cache_size=4092, tol=1e-5) svc.fit(X_in, y_in) y_out = svc.predict(X_out) return y_out
def test_center_kernel(): """Test that KernelCenterer is equivalent to Scaler in feature space""" X_fit = np.random.random((5, 4)) scaler = Scaler(with_std=False) scaler.fit(X_fit) X_fit_centered = scaler.transform(X_fit) K_fit = np.dot(X_fit, X_fit.T) # center fit time matrix centerer = KernelCenterer() K_fit_centered = np.dot(X_fit_centered, X_fit_centered.T) K_fit_centered2 = centerer.fit_transform(K_fit) assert_array_almost_equal(K_fit_centered, K_fit_centered2) # center predict time matrix X_pred = np.random.random((2, 4)) K_pred = np.dot(X_pred, X_fit.T) X_pred_centered = scaler.transform(X_pred) K_pred_centered = np.dot(X_pred_centered, X_fit_centered.T) K_pred_centered2 = centerer.transform(K_pred) assert_array_almost_equal(K_pred_centered, K_pred_centered2)
def run_real_data_experiments(nr_samples, delta, verbose=0, do_scatter_plot=False): dataset = Dataset('hollywood2', suffix='.per_slice.delta_%d' % delta, nr_clusters=256) samples, _ = dataset.get_data('test') nr_samples = np.minimum(len(samples), nr_samples) nr_samples = np.maximum(1, nr_samples) if verbose > 2: print "Loading train data." tr_data, _, _ = load_sample_data(dataset, 'train', pi_derivatives=True) scaler = Scaler() scaler.fit(tr_data) true_values, approx_values = [], [] for ii in xrange(nr_samples): if verbose > 2: sys.stdout.write("%s\r" % samples[ii].movie) data, _, _ = load_sample_data(dataset, str(samples[ii]), pi_derivatives=True) data = scaler.transform(data) L2_norm_true, L2_norm_approx = L2_approx(data) true_values.append(L2_norm_true) approx_values.append(L2_norm_approx) if verbose: print print_info(true_values, approx_values, verbose) print if do_scatter_plot: scatter_plot(true_values, approx_values)
records = data[:,1:] labels = data[:,0] n_train = 35000 #n_val = n - n_train n_val = 7000 trainset = records[:n_train,:] trainlabels = labels[:n_train] #valset = records[n_train:,:] #vallabels = labels[n_train:,:] valset = records[n_train:n_train+n_val,:] vallabels = labels[n_train:n_train+n_val] n,dim = trainset.shape # mean centering, stdev normalization and whitening scaler = Scaler() scaler.fit(trainset) trainset = scaler.transform(trainset) valset = scaler.transform(valset) pca = PCA(n_components=dim,whiten=True) pca.fit(trainset) trainset = pca.transform(trainset) valset = pca.transform(valset) config = Train_config() config.iterations = 10 config.nonlinearity = 'tanh' config.batchsize = 50 config.learning_rate = 0.2 config.momentum = 0.7 log = open('log.txt','w') nn = Net([dim,300,10],log_file=log)
def test_scaler(): """Test scaling of dataset along all axis""" # First test with 1D data X = np.random.randn(5) X_orig_copy = X.copy() scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert_array_almost_equal(X_scaled_back, X_orig_copy) # Test with 1D list X = [0., 1., 2, 0.4, 1.] scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=False) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) X_scaled = scale(X) assert_array_almost_equal(X_scaled.mean(axis=0), 0.0) assert_array_almost_equal(X_scaled.std(axis=0), 1.0) # Test with 2D data X = np.random.randn(4, 5) X[:, 0] = 0.0 # first feature is always of zero scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert X_scaled is not X # check inverse transform X_scaled_back = scaler.inverse_transform(X_scaled) assert X_scaled_back is not X assert X_scaled_back is not X_scaled assert_array_almost_equal(X_scaled_back, X) X_scaled = scale(X, axis=1, with_std=False) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) X_scaled = scale(X, axis=1, with_std=True) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0]) # Check that the data hasn't been modified assert X_scaled is not X X_scaled = scaler.fit(X).transform(X, copy=False) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert X_scaled is X X = np.random.randn(4, 5) X[:, 0] = 1.0 # first feature is a constant, non zero feature scaler = Scaler() X_scaled = scaler.fit(X).transform(X, copy=True) assert not np.any(np.isnan(X_scaled)) assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0]) assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.]) # Check that X has not been copied assert X_scaled is not X
def Logistic_train(X_in, y_in, X_out, cs, file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format( len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breakes the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1 * len(X_in)))) X_scaler = [X_in[i] for i in test_indices] y_scaler = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_scaler, y_scaler) X_scaler = scaler.transform(X_scaler) X_in = scaler.transform(X_in) X_out = scaler.transform( X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scaler.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scaler = [[X_scaler[i][j] for j in f_indices] for i in range(len(X_scaler))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(X_in[0]) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered best_cv_accuracy = 0. best_c = 0. for c in cs: kfold = cross_validation.StratifiedKFold(y_in, k=10) lrc = LogisticRegression(C=c, tol=1e-5) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([X_in[i][:] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([X_in[i][:] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] lrc.fit(X_train, y_train) in_accuracy += lrc.score(X_train, y_train) cv_accuracy += lrc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('C: {}\n'.format(c)) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_c = c best_cv_accuracy = cv_accuracy #Now tests the out of sample error if file_log: file_log.writelines('\nBEST result: E_cv={}, C={}\n'.format( 1. - best_cv_accuracy, best_c)) lrc = LogisticRegression(C=best_c, tol=1e-5) lrc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - lrc.score(X_in, y_in))) file_log.writelines( 'Etest= {}\n'.format(1. - lrc.score(X_scaler, y_scaler))) y_out = lrc.predict(X_out) return y_out
def SVM_train(X_in, y_in, X_out, gammas, cs, file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format( len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breaks the input set into train. cross validation #and scale sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests scale_set_indices, train_indices = split_indices( len(X_in), int(round(0.1 * len(X_in)))) # shuffle(X_in, y_in) X_scale = [X_in[i] for i in scale_set_indices] y_scale = [y_in[i] for i in scale_set_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #Scale data first scaler = Scaler(copy=False) #WARNING: copy=False => in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on a separate subsetonly, and then reported on data scaler.fit(X_scale, y_scale) X_scale = scaler.transform(X_scale) X_in = scaler.transform(X_in) X_out = scaler.transform( X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scale.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scale = [[X_scale[i][j] for j in f_indices] for i in range(len(X_scale))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] if file_log: file_log.writelines('Initial features :{}, Features used: {}\n'.format( M, len(X_in[0]))) M = len(f_indices) best_cv_accuracy = 0. best_gamma = 0. best_c = 0. #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered for c in cs: for g in gammas: #Balanced cross validation (keeps the ratio of the two classes as #constant as possible across the k folds). kfold = cross_validation.StratifiedKFold(y_in, k=10) svc = svm.SVC(kernel='rbf', C=c, gamma=g, verbose=False, cache_size=4092, tol=1e-5) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([X_in[i][:] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([X_in[i][:] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] svc.fit(X_train, y_train) in_accuracy += svc.score(X_train, y_train) cv_accuracy += svc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('C:{}, gamma:{}\n'.format(c, g)) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_gamma = g best_c = c best_cv_accuracy = cv_accuracy if file_log: file_log.writelines('\nBEST result: E_cv={}, C={}, gamma={}\n'.format( 1. - best_cv_accuracy, best_c, best_gamma)) svc = svm.SVC(kernel='rbf', C=best_c, gamma=best_gamma, verbose=False, cache_size=4092, tol=1e-5) svc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - svc.score(X_in, y_in))) file_log.writelines('Etest= {}\n'.format(1. - svc.score(X_scale, y_scale))) y_out = svc.predict(X_out) #DEBUG: output = ['{} {:+}\n'.format(id_out[i], int(y_scale[i])) for i in range(len(X_out))] #DEBUG: file_log.writelines('------------------------') return y_out
def tree_train(X_in, y_in, X_out, min_meaningful_features_ratio=1., file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format(len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breaks the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1*len(X_in)))) X_scaler = [X_in[i] for i in test_indices] y_scaler = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_scaler, y_scaler) X_scaler = scaler.transform(X_scaler) X_in = scaler.transform(X_in) X_out = scaler.transform(X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scaler.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scaler = [[X_scaler[i][j] for j in f_indices] for i in range(len(X_scaler))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(f_indices) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered best_cv_accuracy = 0. best_features_number = M for features_number in range(int(floor(M * min_meaningful_features_ratio)), M + 1): # kfold = cross_validation.KFold(len(y_in), k=10, shuffle=True) kfold = cross_validation.StratifiedKFold(y_in, k=10) svc = ExtraTreesClassifier(criterion='entropy', max_features=features_number) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([[X_in[i][j] for j in range(M)] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([[X_in[i][j] for j in range(M)] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] svc.fit(X_train, y_train) in_accuracy += svc.score(X_train, y_train) cv_accuracy += svc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('# of features: {}\n'.format(len(X_train[0]))) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_features_number = features_number best_cv_accuracy = cv_accuracy #Now tests the out of sample error if file_log: file_log.writelines('\nBEST result: E_cv={}, t={}\n'.format(1. - best_cv_accuracy, best_features_number)) svc = ExtraTreesClassifier(criterion='entropy', n_estimators=features_number) svc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - svc.score(X_in, y_in))) file_log.writelines('Etest= {}\n'.format(1. - svc.score(X_scaler, y_scaler))) y_out = svc.predict(X_out) return y_out
def Logistic_train(X_in, y_in, X_out, cs, file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format(len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breakes the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1*len(X_in)))) X_scaler = [X_in[i] for i in test_indices] y_scaler = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_scaler, y_scaler) X_scaler = scaler.transform(X_scaler) X_in = scaler.transform(X_in) X_out = scaler.transform(X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scaler.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scaler = [[X_scaler[i][j] for j in f_indices] for i in range(len(X_scaler))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(X_in[0]) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered best_cv_accuracy = 0. best_c = 0. for c in cs: kfold = cross_validation.StratifiedKFold(y_in, k=10) lrc = LogisticRegression(C=c, tol=1e-5) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([X_in[i][:] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([X_in[i][:] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] lrc.fit(X_train, y_train) in_accuracy += lrc.score(X_train, y_train) cv_accuracy += lrc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('C: {}\n'.format(c)) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_c = c best_cv_accuracy = cv_accuracy #Now tests the out of sample error if file_log: file_log.writelines('\nBEST result: E_cv={}, C={}\n'.format(1. - best_cv_accuracy, best_c)) lrc = LogisticRegression(C=best_c, tol=1e-5) lrc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - lrc.score(X_in, y_in))) file_log.writelines('Etest= {}\n'.format(1. - lrc.score(X_scaler, y_scaler))) y_out = lrc.predict(X_out) return y_out
def SVM_train(X_in, y_in, X_out, gammas, cs, file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format(len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breaks the input set into train. cross validation #and scale sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests scale_set_indices, train_indices = split_indices(len(X_in), int(round(0.1*len(X_in)))) # shuffle(X_in, y_in) X_scale = [X_in[i] for i in scale_set_indices] y_scale = [y_in[i] for i in scale_set_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #Scale data first scaler = Scaler(copy=False) #WARNING: copy=False => in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on a separate subsetonly, and then reported on data scaler.fit(X_scale, y_scale) X_scale = scaler.transform(X_scale) X_in = scaler.transform(X_in) X_out = scaler.transform(X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scale.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scale = [[X_scale[i][j] for j in f_indices] for i in range(len(X_scale))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] if file_log: file_log.writelines('Initial features :{}, Features used: {}\n'.format(M, len(X_in[0]))) M = len(f_indices) best_cv_accuracy = 0. best_gamma = 0. best_c = 0. #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered for c in cs: for g in gammas: #Balanced cross validation (keeps the ratio of the two classes as #constant as possible across the k folds). kfold = cross_validation.StratifiedKFold(y_in, k=10) svc = svm.SVC(kernel='rbf', C=c, gamma=g, verbose=False, cache_size=4092, tol=1e-5) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([X_in[i][:] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([X_in[i][:] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] svc.fit(X_train, y_train) in_accuracy += svc.score(X_train, y_train) cv_accuracy += svc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('C:{}, gamma:{}\n'.format(c, g)) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_gamma = g best_c = c best_cv_accuracy = cv_accuracy if file_log: file_log.writelines('\nBEST result: E_cv={}, C={}, gamma={}\n'.format(1. - best_cv_accuracy, best_c, best_gamma)) svc = svm.SVC(kernel='rbf', C=best_c, gamma=best_gamma, verbose=False, cache_size=4092, tol=1e-5) svc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - svc.score(X_in, y_in))) file_log.writelines('Etest= {}\n'.format(1. - svc.score(X_scale, y_scale))) y_out = svc.predict(X_out) #DEBUG: output = ['{} {:+}\n'.format(id_out[i], int(y_scale[i])) for i in range(len(X_out))] #DEBUG: file_log.writelines('------------------------') return y_out
def tree_train(X_in, y_in, X_out, min_meaningful_features_ratio=1., file_log=None): if file_log: file_log.writelines('# of Samples: {}, # of Features: {}\n'.format( len(X_in), len(X_in[0]))) M = len(X_in[0]) #Number of features seed(time()) #To prevent data snooping, breaks the input set into train. cross validation and test sets, with sizes proportional to 8-1-1 #First puts aside 10% of the data for the tests test_indices, train_indices = split_indices(len(X_in), int(round(0.1 * len(X_in)))) X_scaler = [X_in[i] for i in test_indices] y_scaler = [y_in[i] for i in test_indices] X_in = [X_in[i] for i in train_indices] y_in = [y_in[i] for i in train_indices] #scale data first scaler = Scaler(copy=False) #in place modification #Normalize the data and stores as inner parameters the mean and standard deviation #To avoid data snooping, normalization is computed on training set only, and then reported on data scaler.fit(X_scaler, y_scaler) X_scaler = scaler.transform(X_scaler) X_in = scaler.transform(X_in) X_out = scaler.transform( X_out) #uses the same transformation (same mean_ and std_) fit before std_test = X_scaler.std(axis=0) f_indices = [j for j in range(M) if std_test[j] > 1e-7] #Removes feature with null variance X_in = [[X_in[i][j] for j in f_indices] for i in range(len(X_in))] X_scaler = [[X_scaler[i][j] for j in f_indices] for i in range(len(X_scaler))] X_out = [[X_out[i][j] for j in f_indices] for i in range(len(X_out))] M = len(f_indices) #Then, on the remaining data, performs a ten-fold cross validation over the number of features considered best_cv_accuracy = 0. best_features_number = M for features_number in range(int(floor(M * min_meaningful_features_ratio)), M + 1): # kfold = cross_validation.KFold(len(y_in), k=10, shuffle=True) kfold = cross_validation.StratifiedKFold(y_in, k=10) svc = ExtraTreesClassifier(criterion='entropy', max_features=features_number) in_accuracy = 0. cv_accuracy = 0. for t_indices, cv_indices in kfold: X_train = array([[X_in[i][j] for j in range(M)] for i in t_indices]) y_train = [y_in[i] for i in t_indices] X_cv = array([[X_in[i][j] for j in range(M)] for i in cv_indices]) y_cv = [y_in[i] for i in cv_indices] svc.fit(X_train, y_train) in_accuracy += svc.score(X_train, y_train) cv_accuracy += svc.score(X_cv, y_cv) in_accuracy /= kfold.k cv_accuracy /= kfold.k if file_log: file_log.writelines('# of features: {}\n'.format(len(X_train[0]))) file_log.writelines('\tEin= {}\n'.format(1. - in_accuracy)) file_log.writelines('\tEcv= {}\n'.format(1. - cv_accuracy)) if (cv_accuracy > best_cv_accuracy): best_features_number = features_number best_cv_accuracy = cv_accuracy #Now tests the out of sample error if file_log: file_log.writelines('\nBEST result: E_cv={}, t={}\n'.format( 1. - best_cv_accuracy, best_features_number)) svc = ExtraTreesClassifier(criterion='entropy', n_estimators=features_number) svc.fit(X_in, y_in) if file_log: file_log.writelines('Ein= {}\n'.format(1. - svc.score(X_in, y_in))) file_log.writelines( 'Etest= {}\n'.format(1. - svc.score(X_scaler, y_scaler))) y_out = svc.predict(X_out) return y_out
all_folds[split, fold, train] = 1 all_folds[split, fold, test] = 0 for d in range(0, dims.shape[0]): Xtrain = Xm_shfl[train, :, dims[d]] ytrain = y_shfl[train] sw_train = sw_shfl[train] # (deal with NaN in training) ytrain = ytrain[~np.isnan(np.nansum(Xtrain, axis=1))] sw_train = sw_train[~np.isnan(np.nansum(Xtrain, axis=1))] Xtrain = Xtrain[~np.isnan(np.nansum(Xtrain, axis=1)), :] if np.unique(ytrain).shape[0] > 1: # feature selection (find the 50% most discriminative channels) fs.fit(Xtrain, ytrain) # find Xtrain = fs.transform(Xtrain) # remove unnecessary channels # normalization scaler.fit(Xtrain) # find Xtrain = scaler.transform(Xtrain) # apply zscore # SVM fit clf.fit(Xtrain, ytrain, sample_weight=sw_train) # retrieve hyperplan feature identification coef[split, fold, dims[d], :, :] = 0 # initialize #--- univariate uni_features = fs.pvalues_ <= stats.scoreatpercentile(fs.pvalues_, fs.percentile) #--- multivariate coef[split, fold, dims[d], :, uni_features] = clf.coef_.T # predict cross val (deal with NaN in testing) Xtest = Xm_shfl[test, :, dims[d]] test_nan = np.isnan(np.nansum(Xtest, axis=1)) Xtest = fs.transform(Xtest) Xtest = scaler.transform(Xtest) if (Xtest.shape[0] - np.sum(test_nan)) > 0: