def test_multinomial(): """Test all multinomial functionality""" glm_mn = GLM(distr='multinomial', reg_lambda=np.array([0.0, 0.1, 0.2]), learning_rate=2e-1, tol=1e-10) X = np.array([[-1, -2, -3], [4, 5, 6]]) y = np.array([1, 0]) # test gradient beta = np.zeros([4, 2]) grad_beta0, grad_beta = glm_mn._grad_L2loss(beta[0], beta[1:], 0, X, y) assert_true(grad_beta0[0] != grad_beta0[1]) glm_mn.fit(X, y) y_pred = glm_mn.predict(X) assert_equal(y_pred.shape, (3, X.shape[0], 2)) # n_lambdas x n_samples x n_classes # pick one as yhat yhat = y_pred[0] # uniform prediction ynull = np.ones(yhat.shape) / yhat.shape[1] # pseudo_R2 should be greater than 0 assert_true(glm_mn.score(y, yhat, ynull, method='pseudo_R2') > 0.) glm_mn.score(y, yhat) assert_equal( len(glm_mn.simulate(glm_mn.fit_[0]['beta0'], glm_mn.fit_[0]['beta'], X)), X.shape[0]) # these should raise an exception assert_raises(ValueError, glm_mn.score, y, y, y, 'pseudo_R2') assert_raises(ValueError, glm_mn.score, y, y, None, 'deviance')
def test_group_lasso(): """Group Lasso test.""" n_samples, n_features = 100, 90 # assign group ids groups = np.zeros(90) groups[0:29] = 1 groups[30:59] = 2 groups[60:] = 3 # sample random coefficients beta0 = np.random.normal(0.0, 1.0, 1) beta = np.random.normal(0.0, 1.0, n_features) beta[groups == 2] = 0. # create an instance of the GLM class glm_group = GLM(distr='softplus', alpha=1.) # simulate training data Xr = np.random.normal(0.0, 1.0, [n_samples, n_features]) yr = simulate_glm(glm_group.distr, beta0, beta, Xr) # scale and fit scaler = StandardScaler().fit(Xr) glm_group.fit(scaler.transform(Xr), yr)
def glm_bernoulli_pyglmnet(Xr, Yr, Xt): #poissonexp isn't listed as an option for distr? #glm = GLM(distr='poissonexp', alpha=0., reg_lambda=[0.], tol=1e-6) glm = GLM(distr='binomial', alpha=0., reg_lambda=[0.], tol=1e-6) glm.fit(Xr, Yr) Yt = glm.predict(Xt)[0] return Yt
def test_tikhonov(): """Tikhonov regularization test.""" n_samples, n_features = 100, 10 # design covariance matrix of parameters Gam = 15. PriorCov = np.zeros([n_features, n_features]) for i in np.arange(0, n_features): for j in np.arange(i, n_features): PriorCov[i, j] = np.exp(-Gam * 1. / (np.float(n_features) ** 2) * (np.float(i) - np.float(j)) ** 2) PriorCov[j, i] = PriorCov[i, j] if i == j: PriorCov[i, j] += 0.01 PriorCov = 1. / np.max(PriorCov) * PriorCov # sample parameters as multivariate normal beta0 = np.random.randn() beta = np.random.multivariate_normal(np.zeros(n_features), PriorCov) # sample train and test data glm_sim = GLM(distr='softplus', score_metric='pseudo_R2') X = np.random.randn(n_samples, n_features) y = simulate_glm(glm_sim.distr, beta0, beta, X) from sklearn.cross_validation import train_test_split Xtrain, Xtest, ytrain, ytest = \ train_test_split(X, y, test_size=0.5, random_state=42) # design tikhonov matrix [U, S, V] = np.linalg.svd(PriorCov, full_matrices=False) Tau = np.dot(np.diag(1. / np.sqrt(S)), U) Tau = 1. / np.sqrt(np.float(n_samples)) * Tau / Tau.max() # fit model with batch gradient glm_tikhonov = GLM(distr='softplus', alpha=0.0, Tau=Tau, solver='batch-gradient', tol=1e-5, score_metric='pseudo_R2') glm_tikhonov.fit(Xtrain, ytrain) R2_train, R2_test = dict(), dict() R2_train['tikhonov'] = glm_tikhonov.score(Xtrain, ytrain) R2_test['tikhonov'] = glm_tikhonov.score(Xtest, ytest) # fit model with cdfast glm_tikhonov = GLM(distr='softplus', alpha=0.0, Tau=Tau, solver='cdfast', tol=1e-5, score_metric='pseudo_R2') glm_tikhonov.fit(Xtrain, ytrain) R2_train, R2_test = dict(), dict() R2_train['tikhonov'] = glm_tikhonov.score(Xtrain, ytrain) R2_test['tikhonov'] = glm_tikhonov.score(Xtest, ytest)
def test_cv(): """Simple CV check""" X, y = make_regression() model_mn = GLM(distr='normal', alpha=0.01, reg_lambda=np.array([0.0, 0.1, 0.2])) model_mn.fit(X, y) cv = KFold(X.shape[0], 5) # check that it returns 5 scores assert_equal(len(cross_val_score(model_mn, X, y, cv=cv, scoring=simple_cv_scorer)), 5)
def test_multinomial(): """Test all multinomial functionality""" glm_mn = GLM(distr='multinomial', reg_lambda=np.array([0.0, 0.1, 0.2]), learning_rate = 2e-1, tol=1e-10) X = np.array([[-1, -2, -3], [4, 5, 6]]) y = np.array([1, 0]) # test gradient beta = np.zeros([4, 2]) grad_beta0, grad_beta = glm_mn._grad_L2loss(beta[0], beta[1:], 0, X, y) assert_true(grad_beta0[0] != grad_beta0[1]) glm_mn.fit(X, y) y_pred = glm_mn.predict(X) assert_equal(y_pred.shape, (3, X.shape[0], 2)) # n_lambdas x n_samples x n_classes # pick one as yhat yhat = y_pred[0] # uniform prediction ynull = np.ones(yhat.shape) / yhat.shape[1] # pseudo_R2 should be greater than 0 assert_true(glm_mn.score(y, yhat, ynull, method='pseudo_R2') > 0.) glm_mn.score(y, yhat) assert_equal(len(glm_mn.simulate(glm_mn.fit_[0]['beta0'], glm_mn.fit_[0]['beta'], X)), X.shape[0]) # these should raise an exception assert_raises(ValueError, glm_mn.score, y, y, y, 'pseudo_R2') assert_raises(ValueError, glm_mn.score, y, y, None, 'deviance')
def test_cdfast(distr): """Test all functionality related to fast coordinate descent.""" scaler = StandardScaler() n_samples = 1000 n_features = 100 n_classes = 5 density = 0.1 # Batch gradient not available for gamma if distr == 'gamma': return glm = GLM(distr, solver='cdfast') np.random.seed(glm.random_state) # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray()[:, 0] # data X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) y = simulate_glm(glm.distr, beta0, beta, X) # compute grad and hess beta_ = np.zeros((n_features + 1,)) beta_[0] = beta0 beta_[1:] = beta z = beta_[0] + np.dot(X, beta_[1:]) k = 1 xk = X[:, k - 1] gk, hk = _gradhess_logloss_1d(glm.distr, xk, y, z, glm.eta, glm.theta) # test grad and hess if distr != 'multinomial': assert(np.size(gk) == 1) assert(np.size(hk) == 1) assert(isinstance(gk, float)) assert(isinstance(hk, float)) else: assert(gk.shape[0] == n_classes) assert(hk.shape[0] == n_classes) assert(isinstance(gk, np.ndarray)) assert(isinstance(hk, np.ndarray)) assert(gk.ndim == 1) assert(hk.ndim == 1) # test cdfast ActiveSet = np.ones(n_features + 1) beta_ret = glm._cdfast(X, y, ActiveSet, beta_, glm.reg_lambda) assert(beta_ret.shape == beta_.shape) assert(True not in np.isnan(beta_ret))
def test_cdfast(): """Test all functionality related to fast coordinate descent""" scaler = StandardScaler() n_samples = 1000 n_features = 100 n_classes = 5 density = 0.1 distrs = ['softplus', 'gaussian', 'binomial', 'poisson', 'probit'] for distr in distrs: glm = GLM(distr, solver='cdfast') np.random.seed(glm.random_state) # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray()[:, 0] # data X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) y = simulate_glm(glm.distr, beta0, beta, X) # compute grad and hess beta_ = np.zeros((n_features + 1,)) beta_[0] = beta0 beta_[1:] = beta z = beta_[0] + np.dot(X, beta_[1:]) k = 1 xk = X[:, k - 1] gk, hk = glm._gradhess_logloss_1d(xk, y, z) # test grad and hess if distr != 'multinomial': assert_equal(np.size(gk), 1) assert_equal(np.size(hk), 1) assert_true(isinstance(gk, float)) assert_true(isinstance(hk, float)) else: assert_equal(gk.shape[0], n_classes) assert_equal(hk.shape[0], n_classes) assert_true(isinstance(gk, np.ndarray)) assert_true(isinstance(hk, np.ndarray)) assert_equal(gk.ndim, 1) assert_equal(hk.ndim, 1) # test cdfast ActiveSet = np.ones(n_features + 1) beta_ret, z_ret = glm._cdfast(X, y, z, ActiveSet, beta_, glm.reg_lambda) assert_equal(beta_ret.shape, beta_.shape) assert_equal(z_ret.shape, z.shape)
def test_cdfast(): """Test all functionality related to fast coordinate descent""" scaler = StandardScaler() n_samples = 1000 n_features = 100 n_classes = 5 density = 0.1 distrs = ['softplus', 'gaussian', 'binomial', 'poisson', 'probit'] for distr in distrs: glm = GLM(distr, solver='cdfast') np.random.seed(glm.random_state) # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray()[:, 0] # data X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) y = simulate_glm(glm.distr, beta0, beta, X) # compute grad and hess beta_ = np.zeros((n_features + 1,)) beta_[0] = beta0 beta_[1:] = beta z = beta_[0] + np.dot(X, beta_[1:]) k = 1 xk = X[:, k - 1] gk, hk = _gradhess_logloss_1d(glm.distr, xk, y, z, glm.eta) # test grad and hess if distr != 'multinomial': assert_equal(np.size(gk), 1) assert_equal(np.size(hk), 1) assert_true(isinstance(gk, float)) assert_true(isinstance(hk, float)) else: assert_equal(gk.shape[0], n_classes) assert_equal(hk.shape[0], n_classes) assert_true(isinstance(gk, np.ndarray)) assert_true(isinstance(hk, np.ndarray)) assert_equal(gk.ndim, 1) assert_equal(hk.ndim, 1) # test cdfast ActiveSet = np.ones(n_features + 1) beta_ret, z_ret = glm._cdfast(X, y, z, ActiveSet, beta_, glm.reg_lambda) assert_equal(beta_ret.shape, beta_.shape) assert_equal(z_ret.shape, z.shape)
def test_cv(): """Simple CV check.""" # XXX: don't use scikit-learn for tests. X, y = make_regression() glm_normal = GLM(distr='gaussian', alpha=0.01, reg_lambda=0.1) glm_normal.fit(X, y) cv = KFold(X.shape[0], 5) # check that it returns 5 scores assert_equal(len(cross_val_score(glm_normal, X, y, cv=cv, scoring=simple_cv_scorer)), 5)
def test_cv(): """Simple CV check""" # XXX: don't use scikit-learn for tests. X, y = make_regression() glm_normal = GLM(distr='gaussian', alpha=0.01, reg_lambda=[0.0, 0.1, 0.2]) glm_normal.fit(X, y) cv = KFold(X.shape[0], 5) # check that it returns 5 scores assert_equal(len(cross_val_score(glm_normal, X, y, cv=cv, scoring=simple_cv_scorer)), 5)
def test_pseudoR2(): """Test pseudo r2.""" n_samples, n_features = 1000, 100 beta0 = np.random.rand() beta = np.random.normal(0.0, 1.0, n_features) # sample train and test data glm_sim = GLM(score_metric='pseudo_R2') X = np.random.randn(n_samples, n_features) y = simulate_glm(glm_sim.distr, beta0, beta, X) glm_sim.fit(X, y) score = glm_sim.score(X, y) assert (isinstance(score, float))
def test_gradients(distr): """Test gradient accuracy.""" # data scaler = StandardScaler() n_samples, n_features = 1000, 100 X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) density = 0.1 beta_ = np.zeros(n_features + 1) beta_[0] = np.random.rand() beta_[1:] = sps.rand(n_features, 1, density=density).toarray()[:, 0] reg_lambda = 0.1 glm = GLM(distr=distr, reg_lambda=reg_lambda) y = simulate_glm(glm.distr, beta_[0], beta_[1:], X) func = partial(_L2loss, distr, glm.alpha, glm.Tau, reg_lambda, X, y, glm.eta, glm.theta, glm.group) grad = partial(_grad_L2loss, distr, glm.alpha, glm.Tau, reg_lambda, X, y, glm.eta, glm.theta) approx_grad = approx_fprime(beta_, func, 1.5e-8) analytical_grad = grad(beta_) assert_allclose(approx_grad, analytical_grad, rtol=1e-5, atol=1e-3)
def test_deviance(): """Test deviance.""" n_samples, n_features = 1000, 100 beta0 = np.random.normal(0.0, 1.0, 1) beta = np.random.normal(0.0, 1.0, n_features) # sample train and test data glm_sim = GLM(score_metric='deviance') X = np.random.randn(n_samples, n_features) y = simulate_glm(glm_sim.distr, beta0, beta, X) glm_sim.fit(X, y) score = glm_sim.score(X, y) assert_true(isinstance(score, float))
def test_accuracy(): """Testing accuracy.""" n_samples, n_features, n_classes = 1000, 100, 2 beta0 = np.random.normal(0.0, 1.0, 1) beta = np.random.normal(0.0, 1.0, (n_features, n_classes)) # sample train and test data glm_sim = GLM(distr='binomial', score_metric='accuracy') X = np.random.randn(n_samples, n_features) y = simulate_glm(glm_sim.distr, beta0, beta, X) y = np.argmax(y, axis=1) glm_sim.fit(X, y) score = glm_sim.score(X, y) assert_true(isinstance(score, float))
def test_random_state_consistency(): """Test model's random_state.""" # Generate the dataset n_samples, n_features = 1000, 10 beta0 = 1. / (np.float(n_features) + 1.) * np.random.normal(0.0, 1.0) beta = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0, (n_features,)) Xtrain = np.random.normal(0.0, 1.0, [n_samples, n_features]) ytrain = simulate_glm("gaussian", beta0, beta, Xtrain, sample=False, random_state=42) # Test simple glm glm_a = GLM(distr="gaussian", random_state=1) ypred_a = glm_a.fit_predict(Xtrain, ytrain) glm_b = GLM(distr="gaussian", random_state=1) ypred_b = glm_b.fit_predict(Xtrain, ytrain) match = "This glm object has already been fit" with pytest.raises(ValueError, match=match): ypred_c = glm_b.fit_predict(Xtrain, ytrain) # Consistency between two different models assert_array_equal(ypred_a, ypred_b) # Test also cross-validation glm_cv_a = GLMCV(distr="gaussian", cv=3, random_state=1) ypred_a = glm_cv_a.fit_predict(Xtrain, ytrain) glm_cv_b = GLMCV(distr="gaussian", cv=3, random_state=1) ypred_b = glm_cv_b.fit_predict(Xtrain, ytrain) ypred_c = glm_cv_b.fit_predict(Xtrain, ytrain) assert_array_equal(ypred_a, ypred_b) assert_array_equal(ypred_b, ypred_c)
def test_multinomial(): """Test all multinomial functionality""" glm_mn = GLM(distr='multinomial', reg_lambda=np.array([0.0, 0.1, 0.2]), learning_rate = 2e-1, tol=1e-10) X = np.array([[-1, -2, -3], [4, 5, 6]]) y = np.array([1, 0]) # test gradient beta = np.zeros([4, 2]) grad_beta0, grad_beta = glm_mn._grad_L2loss(beta[0], beta[1:], 0, X, y) assert_true(grad_beta0[0] != grad_beta0[1]) glm_mn.fit(X, y) y_pred_proba = glm_mn.predict_proba(X) assert_equal(y_pred_proba.shape, (3, X.shape[0], 2)) # n_lambdas x n_samples x n_classes # pick one as yhat yhat = y_pred_proba[0] # uniform prediction ynull = np.ones(yhat.shape) / yhat.shape[1] # pseudo_R2 should be greater than 0 assert_true(glm_mn[-1].score(X, y) > 0.) assert_equal(len(glm_mn.simulate(glm_mn.fit_[0]['beta0'], glm_mn.fit_[0]['beta'], X)), X.shape[0]) # check that score is computed for sliced estimator scorelist = glm_mn[-1].score(X, y) assert_equal(scorelist.shape[0], 1) # check that score is computed for all lambdas scorelist = glm_mn.score(X, y) assert_equal(scorelist.shape[0], y_pred_proba.shape[0])
def test_group_lasso(): """Group Lasso test.""" n_samples, n_features = 100, 90 # assign group ids groups = np.zeros(90) groups[0:29] = 1 groups[30:59] = 2 groups[60:] = 3 # sample random coefficients beta0 = np.random.normal(0.0, 1.0, 1) beta = np.random.normal(0.0, 1.0, n_features) beta[groups == 2] = 0. # create an instance of the GLM class glm_group = GLM(distr='softplus', alpha=1., reg_lambda=0.2, group=groups) # simulate training data np.random.seed(glm_group.random_state) Xr = np.random.normal(0.0, 1.0, [n_samples, n_features]) yr = simulate_glm(glm_group.distr, beta0, beta, Xr) # scale and fit scaler = StandardScaler().fit(Xr) glm_group.fit(scaler.transform(Xr), yr) # count number of nonzero coefs for each group. # in each group, coef must be [all nonzero] or [all zero]. beta = glm_group.beta_ group_ids = np.unique(groups) for group_id in group_ids: if group_id == 0: continue target_beta = beta[groups == group_id] n_nonzero = (target_beta != 0.0).sum() assert n_nonzero in (len(target_beta), 0) # one of the groups must be [all zero] assert np.any([ beta[groups == group_id].sum() == 0 for group_id in group_ids if group_id != 0 ])
def test_glmnet(): """Test glmnet.""" glm = GLM(distr='poisson') scaler = StandardScaler() n_samples, n_features = 10000, 100 density = 0.1 # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray() X_train = np.random.normal(0.0, 1.0, [n_samples, n_features]) y_train = glm.simulate(beta0, beta, X_train) X_train = scaler.fit_transform(X_train) glm.fit(X_train, y_train) beta_ = glm.fit_params[-2]['beta'][:] assert_allclose(beta[:], beta_, atol=0.1) # check fit density_ = np.sum(beta_ > 0.1) / float(n_features) assert_allclose(density_, density, atol=0.02) # check density
def test_glmnet(): """Test glmnet.""" scaler = StandardScaler() n_samples, n_features = 100, 10 # coefficients beta0 = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0) beta = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0, (n_features,)) distrs = ['softplus', 'gaussian', 'poisson', 'binomial', 'probit'] solvers = ['batch-gradient', 'cdfast'] score_metric = 'pseudo_R2' learning_rate = 2e-1 for solver in solvers: for distr in distrs: glm = GLM(distr, learning_rate=learning_rate, solver=solver, score_metric=score_metric) assert_true(repr(glm)) np.random.seed(glm.random_state) X_train = np.random.normal(0.0, 1.0, [n_samples, n_features]) y_train = simulate_glm(glm.distr, beta0, beta, X_train) X_train = scaler.fit_transform(X_train) glm.fit(X_train, y_train) beta_ = glm.beta_ assert_allclose(beta, beta_, atol=0.5) # check fit y_pred = glm.predict(scaler.transform(X_train)) assert_equal(y_pred.shape[0], X_train.shape[0]) # test fit_predict glm_poisson = GLM(distr='softplus') glm_poisson.fit_predict(X_train, y_train) assert_raises(ValueError, glm_poisson.fit_predict, X_train[None, ...], y_train)
def buildGLM(data, iters=1000): ''' returns GLM model set to run for iters iterations ''' reg_lambda = np.logspace(np.log(1e-6), np.log(1e-6), 100, base=np.exp(1)) glm_poissonexp = GLM(distr='poisson', verbose=False, alpha=.05, max_iter=iters, learning_rate=2e-1, score_metric='pseudo_R2', reg_lambda=reg_lambda, eta=4.0) return glm_poissonexp
def test_cv(): """Simple CV check.""" # XXX: don't use scikit-learn for tests. X, y = make_regression() cv = KFold(n_splits=5) glm_normal = GLM(distr='gaussian', alpha=0.01, reg_lambda=0.1) # check that it returns 5 scores scores = cross_val_score(glm_normal, X, y, cv=cv) assert(len(scores) == 5) param_grid = [{'alpha': np.linspace(0.01, 0.99, 2)}, {'reg_lambda': np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))}] glmcv = GridSearchCV(glm_normal, param_grid, cv=cv) glmcv.fit(X, y)
def test_multinomial(): """Test all multinomial functionality""" glm_mn = GLM(distr='multinomial', reg_lambda=np.array([0.0, 0.1, 0.2]), learning_rate=2e-1, tol=1e-10) X = np.array([[-1, -2, -3], [4, 5, 6]]) y = np.array([1, 0]) # test gradient beta = np.zeros([4, 2]) grad_beta0, grad_beta = glm_mn._grad_L2loss(beta[0], beta[1:], 0, X, y) assert_true(grad_beta0[0] != grad_beta0[1]) glm_mn.fit(X, y) y_pred_proba = glm_mn.predict_proba(X) assert_equal(y_pred_proba.shape, (3, X.shape[0], 2)) # n_lambdas x n_samples x n_classes # pick one as yhat yhat = y_pred_proba[0] # uniform prediction ynull = np.ones(yhat.shape) / yhat.shape[1] # pseudo_R2 should be greater than 0 assert_true(glm_mn[-1].score(X, y) > 0.) assert_equal( len(glm_mn.simulate(glm_mn.fit_[0]['beta0'], glm_mn.fit_[0]['beta'], X)), X.shape[0]) # check that score is computed for sliced estimator scorelist = glm_mn[-1].score(X, y) assert_equal(scorelist.shape[0], 1) # check that score is computed for all lambdas scorelist = glm_mn.score(X, y) assert_equal(scorelist.shape[0], y_pred_proba.shape[0])
def test_glmnet(): """Test glmnet.""" scaler = StandardScaler() n_samples, n_features = 1000, 100 density = 0.1 n_lambda = 10 # coefficients beta0 = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0) beta = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0, [n_features, 1]) distrs = ['softplus', 'poisson', 'gaussian', 'binomial'] solvers = ['batch-gradient', 'cdfast'] score_metric = 'pseudo_R2' learning_rate = 2e-1 for solver in solvers: for distr in distrs: glm = GLM(distr, learning_rate=learning_rate, solver=solver, score_metric=score_metric) assert_true(repr(glm)) np.random.seed(glm.random_state) X_train = np.random.normal(0.0, 1.0, [n_samples, n_features]) y_train = glm.simulate(beta0, beta, X_train) X_train = scaler.fit_transform(X_train) glm.fit(X_train, y_train) beta_ = glm.fit_[-1]['beta'][:] assert_allclose(beta[:], beta_, atol=0.5) # check fit y_pred = glm.predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (n_lambda, X_train.shape[0])) # checks for slicing. glm = glm[:3] glm_copy = glm.copy() assert_true(glm_copy is not glm) assert_equal(len(glm.reg_lambda), 3) y_pred = glm[:2].predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (2, X_train.shape[0])) y_pred = glm[2].predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (X_train.shape[0], )) assert_raises(IndexError, glm.__getitem__, [2]) glm.score(X_train, y_train) # don't allow slicing if model has not been fit yet. glm_poisson = GLM(distr='softplus') assert_raises(ValueError, glm_poisson.__getitem__, 2) # test fit_predict glm_poisson.fit_predict(X_train, y_train) assert_raises(ValueError, glm_poisson.fit_predict, X_train[None, ...], y_train)
def scca_solution(x, y, x_omega, y_omega, alpha0, beta0, alpha_lambda_ratio, beta_lambda_ratio, alpha_lambda, beta_lambda, niter, eps, glm_impl): """ computes one pair of canonical weights """ for idx in range(niter): x0 = x_omega @ beta0 if glm_impl == 'glmnet_python': from glmnet_python import glmnet lambda_a = np.array([alpha_lambda]) m_ = glmnet(x=y.copy(), y=x0.copy(), standardize=False, intr=False, family='gaussian', lambdau=lambda_a, alpha=alpha_lambda_ratio) alpha1 = m_['beta'][:, -1] elif glm_impl == 'pyglmnet': from pyglmnet import GLM alpha1 = GLM(distr='gaussian', alpha=alpha_lambda_ratio, reg_lambda=alpha_lambda, fit_intercept=False).fit(y.copy(), x0.copy()).beta_ else: raise Exception(str(glm_impl) + ' not supported.') if np.sum(np.abs(alpha1)) < eps: alpha0 = [0]*y.shape[1] break idx_nz = np.where(alpha1 != 0)[0] alpha1_scale = y[:, idx_nz] @ alpha1[idx_nz] alpha1 = alpha1 / np.sqrt(alpha1_scale @ alpha1_scale / (x.shape[0] - 1)) y0 = y_omega @ alpha1 if glm_impl == 'glmnet_python': from glmnet_python import glmnet lambda_b = np.array([beta_lambda]) m_ = glmnet(x=x.copy(), y=y0.copy(), standardize=False, intr=False, family='gaussian', lambdau=lambda_b, alpha=beta_lambda_ratio) beta1 = m_['beta'][:, -1] elif glm_impl == 'pyglmnet': from pyglmnet import GLM beta1 = GLM(distr='gaussian', alpha=beta_lambda_ratio, reg_lambda=beta_lambda, fit_intercept=False).fit(x.copy(), y0.copy()).beta_ else: raise Exception(str(glm_impl) + ' not supported.') if np.sum(np.abs(beta1)) < eps: beta0 = [0]*x.shape[1] break idx_nz = np.where(beta1 != 0)[0] beta1_scale = x[:, idx_nz] @ beta1[idx_nz] beta1 = beta1 / np.sqrt(beta1_scale @ beta1_scale / (x.shape[0] - 1)) if (np.sum(np.abs(alpha1 - alpha0)) < eps and np.sum(np.abs(beta1 - beta0)) < eps): break alpha0 = alpha1 beta0 = beta1 return alpha0, beta0
######################################################## # Now use GridSearchCV to compare import numpy as np # noqa from sklearn.model_selection import GridSearchCV # noqa from sklearn.model_selection import KFold # noqa cv = KFold(3) reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) param_grid = [{'reg_lambda': reg_lambda}] glm = GLM(distr='binomial', alpha=0.05, score_metric='pseudo_R2', learning_rate=0.1, tol=1e-4, verbose=True) glmcv = GridSearchCV(glm, param_grid, cv=cv) glmcv.fit(X_train, y_train) print("test set pseudo $R^2$ = %f" % glmcv.score(X_test, y_test)) ######################################################## # Plot the true and predicted test set target values plt.plot(y_test[:50], 'ko-') plt.plot(y_test_hat[:50], 'ro-') plt.legend(['true', 'pred'], frameon=False) plt.xlabel('Counties') plt.ylabel('Per capita violent crime')
# %% import statsmodels.api as sm mod = sm.GLM(df['cnt'] / df['offset'], df[np.arange(10)], family=sm.families.Poisson()) mod = mod.fit() mod.summary() # %% from pyglmnet import GLM # create an instance of the GLM class glm = GLM(distr='poisson') glm = glm.fit(df[np.arange(10)].values, df['cnt'].values/df['offset'].values) glm # %% glm.get_params() # %% import keras inl = keras.layers.Input((10,)) out = keras.layers.Dense(1, use_bias=False)(inl) out = keras.layers.Lambda(lambda x: keras.backend.exp(x))(out) model = keras.models.Model(inl, out) model.compile(keras.optimizers.Adam(1e-3), 'poisson')
from sklearn.preprocessing import StandardScaler ######################################################## # Import ``GLM`` class from ``pyglmnet`` ######################################################## # import GLM model from pyglmnet import GLM # create regularization parameters for model reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) glm_poissonexp = GLM(distr='poissonexp', verbose=False, alpha=0.05, max_iter=1000, learning_rate=1e-5, reg_lambda=reg_lambda, eta=4.0) ######################################################## # # # .. math:: # # J = \sum_i \lambda_i - y_i \log \lambda_i # # where # # .. math:: #
from copy import deepcopy import matplotlib.pyplot as plt from sklearn.preprocessing import StandardScaler ######################################################## # Import ``GLM`` class from ``pyglmnet`` ######################################################## # import GLM model from pyglmnet import GLM # create regularization parameters for model reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) glm_poissonexp = GLM(distr='poissonexp', verbose=False, alpha=0.05, max_iter=1000, learning_rate=1e-5, reg_lambda=reg_lambda, eta=4.0) ######################################################## # # # .. math:: # # J = \sum_i \lambda_i - y_i \log \lambda_i # # where # # .. math:: # # \lambda_i = # \begin{cases}
# separating the dependent variables from the independent variables X, y = build_Xy(df,'att128') n_features = X.shape[1] n_samples = X.shape[0] #print n_samples #print n_features #Splitting the training and test sets X_train, X_test, y_train, y_test = train_test_split(X,y,train_size=0.5,test_size = 0.5, random_state=0) # Defining the model reg_lambda = np.logspace(np.log(2.0), np.log(0.01), 100, base=np.exp(1)) model = GLM(distr='poisson', verbose=False, alpha=0.05, max_iter=1000, learning_rate=0.001, reg_lambda=reg_lambda, eta=10.0) # Best values for the parameters for R2s: # R2r: 0.714862 # alpha: 0.052 # learning rate: 1e-3 # R2t: 0.674098 # alpha: 0.052 # learning rate: 1e-3 print 'alpha: ', model.alpha print 'learning rate: ', model.learning_rate #model.threshold = 1e-5
# default: 100 ######################################################## ######################################################## # Import ``GLM`` class from ``pyglmnet`` ######################################################## # import GLM model from pyglmnet import GLM # create regularize parameters for model reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) glm_poisson = GLM(distr='poisson', verbose=False, alpha=0.05, max_iter=1000, learning_rate=1e-4, reg_lambda=reg_lambda) ########################################################## # Simulate a dataset # ------------------ # The ``GLM`` class has a very useful method called ``simulate()``. # # Since a canonical link function is already specified by the distribution # parameters, or provided by the user, ``simulate()`` requires # only the independent variables ``X`` and the coefficients ``beta0`` # and ``beta`` ########################################################## n_samples, n_features = 10000, 100
# reading the dataset df = read_dataset('community_crime.csv',0) # separating the dependent variables from the independent variables X, y = build_Xy(df,'att128') n_features = X.shape[1] n_samples = X.shape[0] #Splitting the training and test sets X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.33,random_state=0) # Defining the model model = GLM(distr='multinomial', alpha=0.5, reg_lambda=np.array([0.02, 0.01]), learning_rate=1e-3 ,verbose=False,) #initial values for the coefficients beta0 = np.random.normal(0.0, 1.0, 1) beta = sps.rand(n_features, 1, 0.1) beta = np.array(beta.todense()) model.threshold = 1e-5 #scaler = StandardScaler().fit(X_train) #model.fit(scaler.transform(X_train),y_train) # Fitting the model model.fit(X_train,y_train)
def test_cdfast(): """Test all functionality related to fast coordinate descent""" scaler = StandardScaler() n_samples = 1000 n_features = 100 n_classes = 5 density = 0.1 distrs = ['softplus', 'poisson', 'gaussian', 'binomial', 'multinomial'] for distr in distrs: glm = GLM(distr, solver='cdfast') np.random.seed(glm.random_state) if distr != 'multinomial': # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray() # data X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) y = glm.simulate(beta0, beta, X) elif distr == 'multinomial': # coefficients beta0 = 1 / (n_features + 1) * \ np.random.normal(0.0, 1.0, n_classes) beta = 1 / (n_features + 1) * \ np.random.normal(0.0, 1.0, [n_features, n_classes]) # data X, y = make_classification(n_samples=n_samples, n_features=n_features, n_redundant=0, n_informative=n_features, random_state=1, n_classes=n_classes) y_bk = y.ravel() y = np.zeros([X.shape[0], y.max() + 1]) y[np.arange(X.shape[0]), y_bk] = 1 # compute grad and hess beta_ = np.zeros([n_features + 1, beta.shape[1]]) beta_[0] = beta0 beta_[1:] = beta z = beta_[0] + np.dot(X, beta_[1:]) k = 1 xk = np.expand_dims(X[:, k - 1], axis=1) gk, hk = glm._gradhess_logloss_1d(xk, y, z) # test grad and hess if distr != 'multinomial': assert_equal(np.size(gk), 1) assert_equal(np.size(hk), 1) assert_true(isinstance(gk, float)) assert_true(isinstance(hk, float)) else: assert_equal(gk.shape[0], n_classes) assert_equal(hk.shape[0], n_classes) assert_true(isinstance(gk, np.ndarray)) assert_true(isinstance(hk, np.ndarray)) assert_equal(gk.ndim, 1) assert_equal(hk.ndim, 1) # test cdfast ActiveSet = np.ones(n_features + 1) rl = glm.reg_lambda[0] beta_ret, z_ret = glm._cdfast(X, y, z, ActiveSet, beta_, rl) assert_equal(beta_ret.shape, beta_.shape) assert_equal(z_ret.shape, z.shape)
def test_glmnet(): """Test glmnet.""" scaler = StandardScaler() n_samples, n_features = 10000, 100 density = 0.1 n_lambda = 10 # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray() distrs = ['poisson', 'poissonexp', 'normal', 'binomial'] learning_rate = 2e-1 for distr in distrs: glm = GLM(distr, learning_rate=learning_rate) assert_true(repr(glm)) np.random.seed(glm.random_state) X_train = np.random.normal(0.0, 1.0, [n_samples, n_features]) y_train = glm.simulate(beta0, beta, X_train) X_train = scaler.fit_transform(X_train) glm.fit(X_train, y_train) beta_ = glm.fit_[-2]['beta'][:] assert_allclose(beta[:], beta_, atol=0.5) # check fit density_ = np.sum(beta_ > 0.1) / float(n_features) assert_allclose(density_, density, atol=0.05) # check density y_pred = glm.predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (n_lambda, X_train.shape[0])) # checks for slicing. glm = glm[:3] glm_copy = glm.copy() assert_true(glm_copy is not glm) assert_equal(len(glm.reg_lambda), 3) y_pred = glm[:2].predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (2, X_train.shape[0])) y_pred = glm[2].predict(scaler.transform(X_train)) assert_equal(y_pred.shape, (X_train.shape[0], )) assert_raises(IndexError, glm.__getitem__, [2]) glm.score(y_train, y_pred) # don't allow slicing if model has not been fit yet. glm_poisson = GLM(distr='poisson') assert_raises(ValueError, glm_poisson.__getitem__, 2) # test fit_predict glm_poisson.fit_predict(X_train, y_train) assert_raises(ValueError, glm_poisson.fit_predict, X_train[None, ...], y_train)
######################################################## ######################################################## # Import ``GLM`` class from ``pyglmnet`` ######################################################## # import GLM model from pyglmnet import GLM # create regularization parameters for model reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) glm_poisson = GLM(distr='softplus', verbose=False, alpha=0.05, max_iter=1000, learning_rate=2e-1, reg_lambda=reg_lambda) ########################################################## # Simulate a dataset # ------------------ # The ``GLM`` class has a very useful method called ``simulate()``. # # Since a canonical link function is already specified by the distribution # parameters, or provided by the user, ``simulate()`` requires # only the independent variables ``X`` and the coefficients ``beta0`` # and ``beta`` ##########################################################
n_classes=5, n_informative=100, n_features=100, n_redundant=0) ######################################################## ######################################################## # Fit the model ######################################################## ######################################################## from pyglmnet import GLM glm_mn = GLM(distr='multinomial', alpha=0.01, reg_lambda=np.array([0.02, 0.01]), verbose=False) glm_mn.threshold = 1e-5 glm_mn.fit(X, y) ######################################################## ######################################################## # Predict and score the output ######################################################## y_pred = glm_mn[-1].predict(X).argmax(axis=1) print('Percentage correct = %f percent.' % (y_pred == y).mean()) ########################################################
def test_api_input(): """Test that the input value of y can be of different types.""" random_state = 1 state = np.random.RandomState(random_state) n_samples, n_features = 100, 5 X = state.normal(0, 1, (n_samples, n_features)) y = state.normal(0, 1, (n_samples, )) glm = GLM(distr='gaussian') # Test that ValueError is raised when the shapes mismatch with pytest.raises(ValueError): GLM().fit(X, y[3:]) # This would work without errors glm.fit(X, y) glm.predict(X) glm.score(X, y) glm.plot_convergence() glm = GLM(distr='gaussian', solver='test') with pytest.raises(ValueError, match="solver must be one of"): glm.fit(X, y) with pytest.raises(ValueError, match="fit_intercept must be"): glm = GLM(distr='gaussian', fit_intercept='blah') glm = GLM(distr='gaussian', max_iter=2) with pytest.warns(UserWarning, match='Reached max number of iterat'): glm.fit(X, y)
# - `max_iter`: int (maximum iteration for the model) # default: 1000 ######################################################## ######################################################## # Import ``GLM`` class from ``pyglmnet`` ######################################################## # import GLM model from pyglmnet import GLM # create regularize parameters for model reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1)) glm_poisson = GLM(distr="poisson", verbose=False, alpha=0.05, max_iter=1000, learning_rate=2e-1, reg_lambda=reg_lambda) ########################################################## # Simulate a dataset # ------------------ # The ``GLM`` class has a very useful method called ``simulate()``. # # Since a canonical link function is already specified by the distribution # parameters, or provided by the user, ``simulate()`` requires # only the independent variables ``X`` and the coefficients ``beta0`` # and ``beta`` ########################################################## n_samples, n_features = 10000, 100
def test_cdfast(): """Test all functionality related to fast coordinate descent""" scaler = StandardScaler() n_samples = 1000 n_features = 100 n_classes = 5 density = 0.1 distrs = ['softplus', 'poisson', 'gaussian', 'binomial', 'multinomial'] for distr in distrs: glm = GLM(distr, solver='cdfast') np.random.seed(glm.random_state) if distr != 'multinomial': # coefficients beta0 = np.random.rand() beta = sps.rand(n_features, 1, density=density).toarray() # data X = np.random.normal(0.0, 1.0, [n_samples, n_features]) X = scaler.fit_transform(X) y = glm.simulate(beta0, beta, X) elif distr == 'multinomial': # coefficients beta0 = 1 / (n_features + 1) * \ np.random.normal(0.0, 1.0, n_classes) beta = 1 / (n_features + 1) * \ np.random.normal(0.0, 1.0, [n_features, n_classes]) # data X, y = make_classification(n_samples=n_samples, n_features=n_features, n_redundant=0, n_informative=n_features, random_state=1, n_classes=n_classes) y_bk = y.ravel() y = np.zeros([X.shape[0], y.max() + 1]) y[np.arange(X.shape[0]), y_bk] = 1 # compute grad and hess beta_ = np.zeros([n_features+1, beta.shape[1]]) beta_[0] = beta0 beta_[1:] = beta z = beta_[0] + np.dot(X, beta_[1:]) k = 1 xk = np.expand_dims(X[:, k - 1], axis=1) gk, hk = glm._gradhess_logloss_1d(xk, y, z) # test grad and hess if distr != 'multinomial': assert_equal(np.size(gk), 1) assert_equal(np.size(hk), 1) assert_true(isinstance(gk, float)) assert_true(isinstance(hk, float)) else: assert_equal(gk.shape[0], n_classes) assert_equal(hk.shape[0], n_classes) assert_true(isinstance(gk, np.ndarray)) assert_true(isinstance(hk, np.ndarray)) assert_equal(gk.ndim, 1) assert_equal(hk.ndim, 1) # test cdfast ActiveSet = np.ones(n_features + 1) rl = glm.reg_lambda[0] beta_ret, z_ret = glm._cdfast(X, y, z, ActiveSet, beta_, rl) assert_equal(beta_ret.shape, beta_.shape) assert_equal(z_ret.shape, z.shape)
def test_glmnet(distr, reg_lambda, fit_intercept, solver): """Test glmnet.""" raises(ValueError, GLM, distr='blah') raises(ValueError, GLM, distr='gaussian', max_iter=1.8) n_samples, n_features = 100, 10 # coefficients beta0 = 0. if fit_intercept: beta0 = 1. / (np.float(n_features) + 1.) * \ np.random.normal(0.0, 1.0) beta = 1. / (np.float(n_features) + int(fit_intercept)) * \ np.random.normal(0.0, 1.0, (n_features,)) score_metric = 'pseudo_R2' learning_rate = 2e-1 random_state = 0 betas_ = list() if not (distr == 'gamma' and solver == 'cdfast'): np.random.seed(random_state) theta = 1.0 X_train = np.random.normal(0.0, 1.0, [n_samples, n_features]) y_train = simulate_glm(distr, beta0, beta, X_train, theta=theta, sample=False) alpha = 0. loss_trace = list() eta = 2.0 group = None Tau = None def callback(beta): Tau = None loss_trace.append( _loss(distr, alpha, Tau, reg_lambda, X_train, y_train, eta, theta, group, beta, fit_intercept=fit_intercept)) glm = GLM(distr, learning_rate=learning_rate, reg_lambda=reg_lambda, tol=1e-5, max_iter=5000, alpha=alpha, solver=solver, score_metric=score_metric, random_state=random_state, callback=callback, fit_intercept=fit_intercept, theta=theta) assert(repr(glm)) glm.fit(X_train, y_train) # verify loss decreases assert(np.all(np.diff(loss_trace) <= 1e-7)) # true loss and beta should be recovered when reg_lambda == 0 if reg_lambda == 0.: # verify loss at convergence = loss when beta=beta_ l_true = _loss(distr, alpha, Tau, reg_lambda, X_train, y_train, eta, theta, group, np.concatenate(([beta0], beta))) assert_allclose(loss_trace[-1], l_true, rtol=1e-4, atol=1e-5) # beta=beta_ when reg_lambda = 0. assert_allclose(beta, glm.beta_, rtol=0.05, atol=1e-2) betas_.append(glm.beta_) y_pred = glm.predict(X_train) assert(y_pred.shape[0] == X_train.shape[0]) # compare all solvers pairwise to make sure they're close for i, first_beta in enumerate(betas_[:-1]): for second_beta in betas_[i + 1:]: assert_allclose(first_beta, second_beta, rtol=0.05, atol=1e-2) # test fit_predict glm_poisson = GLM(distr='softplus') glm_poisson.fit_predict(X_train, y_train) raises(ValueError, glm_poisson.fit_predict, X_train[None, ...], y_train)
np.shape(prior_cov) ######################################################## # Fit models from sklearn.cross_validation import train_test_split Xtrain, Xtest, Ytrain, Ytest = train_test_split(features, spike_counts, test_size=0.2, random_state=42) ######################################################## from pyglmnet import utils n_samples = Xtrain.shape[0] Tau = utils.tikhonov_from_prior(prior_cov, n_samples) glm = GLM(distr='poisson', alpha=0., Tau=Tau, score_metric='pseudo_R2') glm.fit(Xtrain, Ytrain) cvopt_lambda = glm.score(Xtest, Ytest).argmax() print("train score: %f" % glm[cvopt_lambda].score(Xtrain, Ytrain)) print("test score: %f" % glm[cvopt_lambda].score(Xtest, Ytest)) weights = glm[cvopt_lambda].fit_['beta'] ######################################################## # Visualize for time_bin_ in range(n_temporal_basis): RF = strf_model.make_image_from_spatial_basis(spatial_basis, weights[range(time_bin_, n_spatial_basis * n_temporal_basis, n_temporal_basis)])
def test_compare_sklearn(solver): """Test results against sklearn.""" def rmse(a, b): return np.sqrt(np.mean((a - b) ** 2)) X, Y, coef_ = make_regression( n_samples=1000, n_features=500, noise=0.1, n_informative=10, coef=True, random_state=42) alpha = 0.1 l1_ratio = 0.5 clf = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, tol=1e-5) clf.fit(X, Y) glm = GLM(distr='gaussian', alpha=l1_ratio, reg_lambda=alpha, solver=solver, tol=1e-6, max_iter=500) glm.fit(X, Y) y_sk = clf.predict(X) y_pg = glm.predict(X) assert abs(rmse(Y, y_sk) - rmse(Y, y_pg)) < 0.5 glm = GLM(distr='gaussian', alpha=l1_ratio, reg_lambda=alpha, solver=solver, tol=1e-6, max_iter=5, fit_intercept=False) glm.fit(X, Y) assert glm.beta0_ == 0. glm.predict(X)
# Download and preprocess data files X, y = datasets.fetch_community_crime_data('/tmp/glm-tools') n_samples, n_features = X.shape ######################################################## # Split the data into training and test sets X_train, X_test, y_train, y_test = \ train_test_split(X, y, test_size=0.33, random_state=0) ######################################################## # Fit a gaussian distributed GLM with elastic net regularization # use the default value for reg_lambda glm = GLM(distr='gaussian', alpha=0.05, score_metric='pseudo_R2') # fit model glm.fit(X_train, y_train) # score the test set prediction y_test_hat = glm[-1].predict(X_test) print ("test set pseudo $R^2$ = %f" % glm[-1].score(X_test, y_test)) ######################################################## # Plot the true and predicted test set target values plt.plot(y_test[:50], 'ko-') plt.plot(y_test_hat[:50], 'ro-') plt.legend(['true', 'pred'], frameon=False) plt.xlabel('Counties')
def test_multinomial(): """Test all multinomial functionality""" glm = GLM(distr='multinomial', reg_lambda=np.array([0.0, 0.1, 0.2]), tol=1e-10) X = np.array([[-1, -2, -3], [4, 5, 6]]) y = np.array([1, 0]) # test gradient beta = np.zeros([4, 2]) grad_beta0, grad_beta = glm.grad_L2loss(beta[0], beta[1:], 0, X, y) assert grad_beta0[0] != grad_beta0[1] glm.fit(X, y) y_pred = glm.predict(X) assert_equal(y_pred.shape, (3, X.shape[0], 2)) # n_lambdas x n_samples x n_classes # pick one as yhat yhat = y_pred[0] # uniform prediction ynull = np.ones(yhat.shape) / yhat.shape[1] # pseudo_R2 should be greater than 0 assert_true(glm.pseudo_R2(y, yhat, ynull) > 0.) glm.deviance(y, yhat) assert_equal(len(glm.simulate(glm.fit_[0]['beta0'], glm.fit_[0]['beta'], X)), X.shape[0]) # these should raise an exception try: glm.pseudo_R2(y, y, y) assert False except Exception: assert True try: glm.deviance(y, y) assert False except Exception: assert True
######################################################## from sklearn.datasets import make_classification X, y = make_classification(n_samples=10000, n_classes=5, n_informative=100, n_features=100, n_redundant=0) ######################################################## ######################################################## # Fit the model ######################################################## ######################################################## from pyglmnet import GLM glm_mn = GLM(distr='multinomial', alpha=0.01, reg_lambda=np.array([0.02, 0.01]), verbose=False) glm_mn.threshold = 1e-5 glm_mn.fit(X, y) ######################################################## ######################################################## # Predict and score the output ######################################################## y_pred = glm_mn[-1].predict(X) print('Percentage correct = %f percent.' % (y_pred == y).mean()) ########################################################
def test_tikhonov(): """Tikhonov regularization test.""" n_samples, n_features = 100, 10 # design covariance matrix of parameters Gam = 15. PriorCov = np.zeros([n_features, n_features]) for i in np.arange(0, n_features): for j in np.arange(i, n_features): PriorCov[i, j] = np.exp(-Gam * 1. / (np.float(n_features) ** 2) * (np.float(i) - np.float(j)) ** 2) PriorCov[j, i] = PriorCov[i, j] if i == j: PriorCov[i, j] += 0.01 PriorCov = 1. / np.max(PriorCov) * PriorCov # sample parameters as multivariate normal beta0 = np.random.randn() beta = np.random.multivariate_normal(np.zeros(n_features), PriorCov) # sample train and test data glm_sim = GLM(distr='softplus', score_metric='pseudo_R2') X = np.random.randn(n_samples, n_features) y = simulate_glm(glm_sim.distr, beta0, beta, X) from sklearn.model_selection import train_test_split Xtrain, Xtest, ytrain, ytest = \ train_test_split(X, y, test_size=0.5, random_state=42) # design tikhonov matrix [U, S, V] = np.linalg.svd(PriorCov, full_matrices=False) Tau = np.dot(np.diag(1. / np.sqrt(S)), U) Tau = 1. / np.sqrt(np.float(n_samples)) * Tau / Tau.max() # fit model with batch gradient glm_tikhonov = GLM(distr='softplus', alpha=0.0, Tau=Tau, solver='batch-gradient', tol=1e-3, score_metric='pseudo_R2') glm_tikhonov.fit(Xtrain, ytrain) R2_train, R2_test = dict(), dict() R2_train['tikhonov'] = glm_tikhonov.score(Xtrain, ytrain) R2_test['tikhonov'] = glm_tikhonov.score(Xtest, ytest) # fit model with cdfast glm_tikhonov = GLM(distr='softplus', alpha=0.0, Tau=Tau, solver='cdfast', tol=1e-3, score_metric='pseudo_R2') glm_tikhonov.fit(Xtrain, ytrain) R2_train, R2_test = dict(), dict() R2_train['tikhonov'] = glm_tikhonov.score(Xtrain, ytrain) R2_test['tikhonov'] = glm_tikhonov.score(Xtest, ytest)
def get_benchmarks(self, X_train, y_train, X_test, y_test): """ """ n_repeats = self.n_repeats distr = self.distr res = dict() for env in self.envs: res[env] = dict() if env == 'pyglmnet': # initialize model model = GLM(distr=distr, reg_lambda=[self.reg_lambda], alpha=self.alpha, solver='batch-gradient', score_metric='pseudo_R2') # fit-predict-score model.fit(X_train, y_train) y_test_hat = model[-1].predict(X_test) y_test_hat = np.squeeze(y_test_hat) if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() model.fit(X_train, y_train) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 if env == 'sklearn': if distr in ['gaussian', 'binomial']: # initialize model if distr == 'gaussian': model = ElasticNet(alpha=self.reg_lambda, l1_ratio=self.alpha) elif distr == 'binomial': model = SGDClassifier(loss='log', penalty='elasticnet', alpha=self.reg_lambda, l1_ratio=self.alpha) # fit-predict-score model.fit(X_train, y_train) y_test_hat = model.predict(X_test) res[env]['score'] = model.score(X_test, y_test) # time tmp = list() for r in range(n_repeats): start = time.time() model.fit(X_train, y_train) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 else: res[env]['score'] = -999. res[env]['time'] = -999. if env == 'statsmodels': # initialize model if distr == 'gaussian': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Gaussian()) elif distr == 'binomial': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Binomial()) elif distr == 'poisson': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Poisson()) # fit-predict-score statsmodels_res = model.fit() y_test_hat = model.predict(statsmodels_res.params, exog=sm.add_constant(X_test)) y_test_hat = np.array(y_test_hat) if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() statsmodels_res = model.fit() stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 if env == 'R': # initialize model glmnet = importr('glmnet') predict = robjects.r('predict') # fit-predict-score try: fit = glmnet.glmnet(X_train, y_train, family=distr, alpha=self.alpha, nlambda=1) tmp = predict(fit, newx=X_test, s=0) y_test_hat = np.zeros(y_test.shape[0]) for i in range(y_test.shape[0]): y_test_hat[i] = tmp[i] if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() fit = glmnet.glmnet(X_train, y_train, family=distr, alpha=self.alpha, nlambda=1) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 except Exception: res[env]['score'] = -999. res[env]['time'] = -999. return res
def fit(self, X, Y, get_history_terms=True): """ Fits the model to the data in X to predict the response Y. Imports models and creates model instance as well. Parameters ---------- X: float, n_samples x n_features, features of interest Y: float, n_samples x 1, population activity get_history_terms = Boolean. Whether to compute the temporal features. Note that if spike_history and cov_history are False, no history will be computed anyways and the flag does nothing. """ if self.default_params: warnings.warn( '\n Using default hyperparameters. Consider optimizing on' + ' a held-out dataset using, e.g. hyperopt or random search') # make the covariate matrix. Include spike or covariate history? # The different methods here are to satisfy the needs of recurrent keras # models if get_history_terms: if self.tunemodel == 'lstm': X, Y = self.get_all_with_history_keras(X, Y) else: X, Y = self.get_all_with_history(X, Y) if self.tunemodel == 'glm': model = GLM(**self.params) model.fit(X, Y) # we want the last of the regularization path self.model = model[-1] elif self.tunemodel == 'feedforward_nn': if np.ndim(X) == 1: X = np.transpose(np.atleast_2d(X)) params = self.params model = Sequential() model.add( Dense(params['n1'], input_dim=np.shape(X)[1], kernel_initializer='glorot_normal', activation='relu', kernel_regularizer=l2(params['l2']))) model.add(Dropout(params['dropout'])) model.add(BatchNormalization()) model.add( Dense(params['n2'], kernel_initializer='glorot_normal', activation='relu', kernel_regularizer=l2(params['l2']))) model.add(BatchNormalization()) model.add(Dense(1, activation='softplus')) optim = adam(lr=params['lr'], clipnorm=params['clipnorm'], decay=params['decay'], beta_1=1 - params['b1'], beta_2=1 - params['b2']) model.compile( loss='poisson', optimizer=optim, ) hist = model.fit(X, Y, batch_size=128, epochs=30, verbose=self.verbose) self.model = model elif self.tunemodel == 'xgboost': dtrain = xgb.DMatrix(X, label=Y) num_round = 200 self.model = xgb.train(self.params, dtrain, num_round) elif self.tunemodel == 'random_forest': self.model = RandomForestRegressor(**self.params) self.model.fit(X, Y) elif self.tunemodel == 'lstm': if np.ndim(X) == 1: X = np.transpose(np.atleast_2d(X)) params = self.params model = Sequential() #Declare model #Add recurrent layer model.add(LSTM(int(params['n_units']),input_shape=(X.shape[1],X.shape[2]),\ dropout_W=params['dropout'],dropout_U=params['dropout'])) #Within recurrent layer, include dropout model.add(Dropout(params['dropout']) ) #Dropout some units (recurrent layer output units) #Add dense connections to output layer model.add(Dense(1, activation='softplus')) #Fit model (and set fitting parameters) model.compile(loss='poisson', optimizer='rmsprop', metrics=['accuracy']) model.fit(X, Y, epochs=int(params['epochs']), batch_size=int(params['batch_size']), verbose=self.verbose) #Fit the model self.model = model else: #using predefined model self.model.fit(X, Y)
def get_benchmarks(self, X_train, y_train, X_test, y_test): """ """ n_repeats = self.n_repeats distr = self.distr res = dict() for env in self.envs: res[env] = dict() if env == 'pyglmnet': # initialize model model = GLM(distr=distr, reg_lambda=[self.reg_lambda], alpha=self.alpha, solver='batch-gradient', score_metric='pseudo_R2') # fit-predict-score model.fit(X_train, y_train) y_test_hat = model[-1].predict(X_test) y_test_hat = np.squeeze(y_test_hat) if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() model.fit(X_train, y_train) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 if env == 'sklearn': if distr in ['gaussian', 'binomial']: # initialize model if distr == 'gaussian': model = ElasticNet(alpha=self.reg_lambda, l1_ratio=self.alpha) elif distr == 'binomial': model = SGDClassifier(loss='log', penalty='elasticnet', alpha=self.reg_lambda, l1_ratio=self.alpha) # fit-predict-score model.fit(X_train, y_train) y_test_hat = model.predict(X_test) res[env]['score'] = model.score(X_test, y_test) # time tmp = list() for r in range(n_repeats): start = time.time() model.fit(X_train, y_train) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 else: res[env]['score'] = -999. res[env]['time'] = -999. if env == 'statsmodels': # initialize model if distr == 'gaussian': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Gaussian()) elif distr == 'binomial': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Binomial()) elif distr == 'poisson': model = sm.GLM(y_train, sm.add_constant(X_train), family=sm.families.Poisson()) # fit-predict-score statsmodels_res = model.fit() y_test_hat = model.predict(statsmodels_res.params, exog=sm.add_constant(X_test)) y_test_hat = np.array(y_test_hat) if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() statsmodels_res = model.fit() stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 if env == 'R': # initialize model glmnet = importr('glmnet') predict = robjects.r('predict') # fit-predict-score try: fit = glmnet.glmnet(X_train, y_train, family=distr, alpha=self.alpha, nlambda=1) tmp = predict(fit, newx=X_test, s=0) y_test_hat = np.zeros(y_test.shape[0]) for i in range(y_test.shape[0]): y_test_hat[i] = tmp[i] if distr in ['gaussian', 'poisson']: res[env]['score'] = \ r2_score(y_test, y_test_hat) elif distr == 'binomial': res[env]['score'] = \ accuracy_score(y_test, (y_test_hat > 0.5).astype(int)) # time tmp = list() for r in range(n_repeats): start = time.time() fit = glmnet.glmnet(X_train, y_train, family=distr, alpha=self.alpha, nlambda=1) stop = time.time() tmp.append(stop - start) res[env]['time'] = np.min(tmp) * 1e3 except: res[env]['score'] = -999. res[env]['time'] = -999. return res