def __init__(self, formula=None, data=None, **kwargs):
# convert all variables raised to a power to float64
# this prevents mis-specification of probabilities in cases of variable overflow
# (if the original var was compressed to a smaller bit integer/float)
if type(data) == pd.DataFrame:
power_vars = list(set(re.findall(r'(?<=power\().+?(?=,)',
formula)))
for var in power_vars:
data[var] = data[var].astype('float64')
if formula:
y, X = patsy.dmatrices(formula, data, 1)
self._y_design_info = y.design_info
self._X_design_info = X.design_info
self._model = GLM(y, X, family=Binomial(), **kwargs)
self._fit = self._model.fit()
self._betas = self._fit.params
self._link = logit
else:
self._y_design_info = None
self._X_design_info = None
self._model = None
self._fit = None
self._betas = None
self._link = logit
def test_binomial(self):
model = BinomialRegressor()
y = Binomial().fitted(self.eta)
model.fit(self.X, y)
y_hat = model.predict(self.X)
diff = y_hat - y
rsq = 1 - np.mean(diff**2) / np.mean((y-np.mean(y))**2)
assert_true(rsq > .9)
def irls_incremental(filename,
formula,
chunksize,
family=Binomial(),
link=Logit(),
maxit=25,
tol=1e-08,
rNames=None,
headerNames=None):
chunck_generator2._filename = filename
chunck_generator2._nRows = chunksize
x = None
y0, dta = incr_dbuilders(
'I(use.eq("Y").mul(1)) ~ age + I(age**2) + urban + livch',
chunck_generator2)
nCols = len(dta.column_names)
if rNames is None:
rNames = dta.column_names
for j in range(1, maxit + 1):
if x is None: x = np.zeros(nCols)
ATWA = np.zeros((nCols, nCols))
ATWz = np.zeros(nCols)
for data_chunk in chunck_generator2():
yb, rowA = dmatrices((y0, dta),
data_chunk,
NA_action="drop",
return_type="dataframe")
yb = yb.values.ravel()
A = np.asarray(rowA[rNames], dtype=np.float32)
eta = np.matmul(A, x)
eta = eta.reshape(len(eta))
g = link.inverse(eta)
mu_eta = link.inverse_deriv
gprime = mu_eta(eta)
z = np.array(eta + (yb - g) / gprime)
varianceFam = family.variance
linkinvFam = link.inverse
g = linkinvFam(eta)
varg = varianceFam(g)
W = gprime**2 / varg
W = W.reshape(len(W), 1)
cross2 = np.matmul(np.transpose(A), np.asarray(W.reshape(-1) * z))
ATWz = ATWz + cross2
cross1 = np.matmul(np.transpose(A), np.asarray(W * A))
ATWA = ATWA + cross1
xold = x
C, rank, piv = cholesky_pivot(ATWA, full_pivot=True)
if rank < C.shape[1]:
raise LinAlgError("Rank-deficiency detected.")
x = solve_triangular(np.transpose(C), ATWz[piv - 1], lower=True)
x = solve_triangular(C, x, lower=False)[piv - 1]
if (np.sqrt(np.matmul(np.transpose(x - xold), x - xold)) < tol): break
if headerNames is not None:
x = pd.DataFrame(x, headerNames)
else:
x = pd.DataFrame(x, index=list(rowA))
return (x, j)
def _train(elements, model_cfg):
"""Construct one model per building block type"""
models = {}
target = model_cfg.target
for model in model_cfg.sections:
# Construct model formula from configuration
terms = " + ".join(["1"] + [f"C({f})" for f in model.factors])
# Train model
models[model.label] = GLM.from_formula(
f"{target} ~ {terms}",
family=Binomial(),
data=filter_data(elements, {model_cfg.label_column: model.label}),
).fit(scale="X2")
return models
def compute_chi2_null_test(model_results, data, dep_var, max_iter, l2_weight):
"""
Compute difference from null model using deviance:
P(null) - P(model) ~ chi_2
"""
null_formula = '%s ~ 1' % (dep_var)
null_model = GLM.from_formula(null_formula,
data,
family=Binomial(link=logit()))
null_model_results = null_model.fit_regularized(maxiter=max_iter,
method='elastic_net',
alpha=l2_weight,
L1_wt=0.0)
model_loglike = model_results.model.loglike(model_results.params)
null_model_loglike = null_model_results.model.loglike(
null_model_results.params)
llr = -2 * (null_model_loglike - model_loglike)
model_df = model_results.model.df_model
p_val = chi2.sf(llr, model_df)
return llr, model_df, p_val
def setUpClass(cls):
cls.forwarding_family = BinomialWrapper()
cls.family = Binomial()
def __init__(self, link=L.logit): # , n=1.):
# TODO: it *should* work for a constant n>1 actually, if data_weights
# is equal to n
self.family = Binomial(link=link)
class BinomialWrapper(FamilyWrapper):
"""
The wrapper of Binomial exponential family distribution,
with function for per sample probability.
Parameters
----------
link : a link instance, optional
The default link for the Binomial family is the logit link.
Available links are logit, probit, cauchy, log, and cloglog.
See statsmodels.family.links for more information.
Attributes
----------
family : a statsmodel Binomial family object
--------
"""
def __init__(self, link=L.logit): # , n=1.):
# TODO: it *should* work for a constant n>1 actually, if data_weights
# is equal to n
self.family = Binomial(link=link)
def loglike_per_sample(self, endog, mu, scale=1.):
"""
The function to calculate log-likelihood per sample
in terms of the fitted mean response.
Parameters
----------
endog : array-like of shape (n, k) or (n, )
Endogenous response variable
mu : array-like of shape (n, )
Fitted mean response variable
scale : float, optional
Not used for the Binomial GLM.
Returns
-------
log_p : array-like of shape (n, )
The value of the loglikelihood function evaluated per sample
(endog,mu,freq_weights,scale) as defined below.
Notes
--------
If the endogenous variable is binary:
.. math::
log_p_{i} = (y_i * \log(\mu_i/(1-\mu_i)) + \log(1-\mu_i))
If the endogenous variable is binomial:
.. math::
log_p_{i} = (\ln \Gamma(n+1) -
\ln \Gamma(y_i + 1) - \ln \Gamma(n_i - y_i +1) + y_i *
\log(\mu_i / (n_i - \mu_i)) + n * \log(1 - \mu_i/n_i))
where :math:`y_i = Y_i * n_i` with :math:`Y_i` and :math:`n_i` as
defined in Binomial initialize. This simply makes :math:`y_i` the
original number of successes.
"""
# special setup
# see _Setup_binomial(self) in generalized_linear_model.py
tmp = self.family.initialize(endog, 1)
endog = tmp[0]
if np.shape(self.family.n) == () and self.family.n == 1:
return scale * (endog * np.log(old_div(mu, (1 - mu)) + 1e-200) +
np.log(1 - mu)).reshape(-1, )
else:
y = endog * self.family.n # convert back to successes
return scale * (special.gammaln(self.family.n + 1) -
special.gammaln(y + 1) -
special.gammaln(self.family.n - y + 1) +
y * np.log(old_div(mu, (1 - mu))) +
self.family.n * np.log(1 - mu)).reshape(-1, )
def update_peaks_fit_regression(data, NE_date_ranges, NE_var, data_name_var,
round_date_var, peak_date_var,
peak_date_buffer, scalar_vars, formula,
max_iter, l2_weight, regression_type):
"""
Randomly update peak times and fit regression.
"""
NE_peak_dates_i = NE_date_ranges.apply(lambda x: np.random.choice(x, 1)[0])
NE_peak_dates_i_df = NE_peak_dates_i.reset_index().rename(
columns={0: peak_date_var})
# data_peak_dates_i = data_date_ranges.apply(lambda x: np.random.choice(x, 1)[0]).reset_index().rename(columns={0 : peak_date_var})
data_i = pd.merge(data,
NE_peak_dates_i_df,
on=[NE_var, data_name_var],
how='inner')
# reassign peaks
data_i = data_i.assign(
**{
'pre_peak': (
data_i.loc[:, round_date_var] <= data_i.loc[:, peak_date_var] -
peak_date_buffer).astype(int),
'post_peak': (
data_i.loc[:, round_date_var] >= data_i.loc[:, peak_date_var] +
peak_date_buffer).astype(int),
'during_peak':
((data_i.loc[:, round_date_var] > data_i.loc[:, peak_date_var] -
peak_date_buffer)
& (data_i.loc[:, round_date_var] < data_i.loc[:, peak_date_var] +
peak_date_buffer)).astype(int),
})
# add days since post-peak
data_i = data_i.assign(
**{
'since_peak':
data_i.loc[:, 'post_peak'] *
(data_i.loc[:, round_date_var] - data_i.loc[:, peak_date_var])
})
# Z-norm all scalar vars
scaler = StandardScaler()
for v in scalar_vars:
data_i = data_i.assign(
**
{v: scaler.fit_transform(data_i.loc[:, v].values.reshape(-1, 1))})
model_full = GLM.from_formula(formula,
data_i,
family=Binomial(link=logit()))
logging.debug(
'%d/%d/%d pre/during/post data' %
(data_i.loc[:, 'pre_peak'].sum(), data_i.loc[:, 'during_peak'].sum(),
data_i.loc[:, 'post_peak'].sum()))
if (regression_type == 'regularized_logit'):
model_res_full = model_full.fit_regularized(maxiter=max_iter,
method='elastic_net',
alpha=l2_weight,
L1_wt=0.0)
model_res_full_err = compute_err_data(model_res_full)
err = model_res_full_err.loc[:, 'SE']
else:
model_res_full = model_full.fit()
err = model_res_full.bse
params = model_res_full.params
return params, err, NE_peak_dates_i
def test_weights(data, dep_var, cat_vars, scalar_vars, l2_weights):
indep_formula = ' + '.join(
['C(%s)' % (cap_cat_var)
for cap_cat_var in cap_cat_vars] + scalar_vars)
formula = '%s ~ %s' % (dep_var, indep_formula)
# convert raw data to exogenous data
# need to do this to force train/test
# to have same features
data_rand = data.copy()
np.random.shuffle(data_rand.values)
model_dummy = GLM.from_formula(formula,
data_rand,
family=Binomial(link=logit()))
exog = model_dummy.exog
exog_names = model_dummy.exog_names
endog = model_dummy.endog
# generate cross validation folds
cross_val_folds = 10
N = data_rand.shape[0]
cross_val_chunk_size = float(N) / cross_val_folds
cross_val_fold_train_idx = [
list(
range(int(floor(i * cross_val_chunk_size)),
int(ceil((i + 1) * cross_val_chunk_size))))
for i in range(cross_val_folds)
]
cross_val_fold_test_idx = [
list(range(0, int(ceil(i * cross_val_chunk_size)))) +
list(range(int(floor((i + 1) * cross_val_chunk_size)), N))
for i in range(cross_val_folds)
]
weight_likelihoods = []
for l2_weight in l2_weights:
print('testing weight = %.3f' % (l2_weight))
likelihoods_l2 = []
for i, (train_idx_i, test_idx_i) in enumerate(
zip(cross_val_fold_train_idx, cross_val_fold_test_idx)):
print('fold %d' % (i))
train_XY = data_rand.iloc[train_idx_i, :]
test_X = exog[test_idx_i, :]
test_Y = endog[test_idx_i]
# fit model
model_i = GLM.from_formula(formula,
train_XY,
family=Binomial(link=logit()))
model_res_i = model_i.fit_regularized(maxiter=max_iter,
method='elastic_net',
alpha=l2_weight,
L1_wt=0.)
# add 0 params for missing coefficients
# to match X shape
model_res_i.params = model_res_i.params.loc[exog_names].fillna(
0, inplace=False)
# score test data
likelihood_i = compute_log_likelihood(model_res_i.params, test_Y,
test_X)
likelihoods_l2.append(likelihood_i)
weight_likelihoods.append(likelihoods_l2)
weight_likelihoods = pd.DataFrame(np.array(weight_likelihoods),
index=l2_weights)
mean_weight_likelihoods = weight_likelihoods.mean(axis=0)
return mean_weight_likelihoods
def run_regression(data,
formula,
regression_type,
dep_var='anchor',
out_dir='../../output',
split_var=None,
split_var_val=0):
"""
Run logit regression on data with given formula
and write to file.
Option: use regularized logit (reduce variable inflation).
:param data: full data
:param formula: regression formula
:param regression_type: type of regression (logit|regularized_logit)
:param dep_var: dependent variable
:param out_dir: output directory
:param split_var: optional variable to split data (e.g. only organization accounts)
:param split_var_val: value of split value variable (if included)
"""
l2_weight = 0.01
max_iter = 100
model_full = GLM.from_formula(formula, data, family=Binomial(link=logit()))
if (regression_type == 'regularized_logit'):
model_res_full = model_full.fit_regularized(maxiter=max_iter,
method='elastic_net',
alpha=l2_weight,
L1_wt=0.0)
else:
model_res_full = model_full.fit()
## summary stats
model_res_full_err = compute_err_data(model_res_full)
# write to file
reg_out_str = 'anchor_%s_output_%s.tsv' % (regression_type,
formula.replace(' ', ''))
if (split_var is not None):
reg_out_str = 'anchor_%s_output_%s_split_%s=%s.tsv' % (
regression_type, formula.replace(' ',
''), split_var, split_var_val)
res_out_file = os.path.join(out_dir, reg_out_str)
model_res_full_err.to_csv(res_out_file, sep='\t', index=True)
## save coeffs to file => pretty print as latex
# need lots of decimal points! for multiple variable correction
pd.options.display.float_format = '{:,.5f}'.format
tex_out_str = reg_out_str.replace('.tsv', '.tex')
tex_res_out_file = os.path.join(out_dir, tex_out_str)
model_res_full_err = model_res_full_err.assign(
**{'coeff': model_res_full_err.index})
tex_data_cols = ['coeff', 'mean', 'SE', 'p_val']
model_res_full_err.to_latex(tex_res_out_file,
columns=tex_data_cols,
index=False)
## compute regression fit parameters => deviance, AIC, etc.
# start with chi2 test against null model
llr, model_df, p_val = compute_chi2_null_test(model_res_full, data,
dep_var, max_iter, l2_weight)
logging.debug('N=%d, LLR=%.5f, df=%d, p-val=%.3E' %
(data.shape[0], llr, model_df, p_val))
# variance inflation factor: are some of the covariates highly collinear?
# for sanity we only look at non-categorical vars
cat_var_matcher = re.compile(
'C\(.+\)\[T\..+\]|Intercept'
) # format="C(var_name)[T.var_val]" ("C(username)[T.barackobama]")
non_cat_params = [
param for param in model_res_full.params.index
if cat_var_matcher.search(param) is None
]
for param in non_cat_params:
VIF_i = compute_VIF(model_res_full, param)
logging.debug('VIF test: param=%s, VIF=%.3f' % (param, VIF_i))
## compute accuracy on k-fold classification
## we would use R-squared but that doesn't work for logistic regression
# first get data into usable format
n_splits = 10
accs = k_fold_acc(model_full.exog, model_full.endog, k=n_splits)
mean_acc = np.mean(accs)
se_acc = np.std(accs) / n_splits**.5
logging.debug('%d-fold mean accuracy = %.3f +/- %.3f' %
(n_splits, mean_acc, se_acc))
def irls_incremental_dm(filename,
chunksize,
yPos=None,
family=Binomial(),
link=Logit(),
maxit=25,
tol=1e-08,
header=None,
headerNames=None):
x = None
nRows = chunksize
tmp = pd.read_csv(filename,
delimiter=',',
header=None,
nrows=1,
parse_dates=[1])
nCols = tmp.shape[1] - 1
if yPos is None:
if header is None:
yPos = nCols
else:
yPos = tmp[nCols].to_string(index=False)[1:]
for j in range(1, maxit + 1):
generator = _generator(filename=filename,
header=header,
chunk_size=nRows)
if x is None: x = np.zeros(nCols)
ATWA = np.zeros((nCols, nCols))
ATWz = np.zeros(nCols)
for rowA in generator:
yb = np.asarray(rowA[yPos].astype(np.float32))
A = np.asarray(rowA.drop([yPos], axis=1), dtype=np.float32)
eta = np.matmul(A, x)
eta = eta.reshape(len(eta))
g = link.inverse(eta)
mu_eta = link.inverse_deriv
gprime = mu_eta(eta)
z = np.array(eta + (yb - g) / gprime)
varianceFam = family.variance
linkinvFam = link.inverse
g = linkinvFam(eta)
varg = varianceFam(g)
W = gprime**2 / varg
W = W.reshape(len(W), 1)
cross2 = np.matmul(np.transpose(A), np.asarray(W.reshape(-1) * z))
ATWz = ATWz + cross2
cross1 = np.matmul(np.transpose(A), np.asarray(W * A))
ATWA = ATWA + cross1
xold = x
C, rank, piv = cholesky_pivot(ATWA, full_pivot=True)
if rank < C.shape[1]:
raise LinAlgError("Rank-deficiency detected.")
x = solve_triangular(np.transpose(C), ATWz[piv - 1], lower=True)
x = solve_triangular(C, x, lower=False)[piv - 1]
if (np.sqrt(np.matmul(np.transpose(x - xold), x - xold)) < tol): break
if headerNames is not None:
x = pd.DataFrame(x, headerNames)
elif header is not None:
x = pd.DataFrame(x, index=list(rowA.drop([yPos], axis=1)))
else:
x = pd.DataFrame(x)
return (x, j)
def glm_svd_newton_dm(X,
y,
family=Binomial(),
link=Logit(),
maxit=25,
tol=1e-08,
stol=1e-08,
singular_ok=True,
weights=None,
reg_method="column projection"):
S = list(svd(X)) # S[0]=u S[1]=d S[2]=v
V = S[2]
nVars = S[2].shape[1]
idx = np.arange(nVars)
i = (S[1] / S[1][0]) > stol
k = np.sum(i)
pivot = np.arange(nVars)
if (k < nVars):
if reg_method == "column projection":
Q, R, pivot = qr(S[2][:, :k], pivoting=True)
idx = np.argsort(pivot[:k])
omit = pivot[-(nvars - k):]
S_new = svd(X[~idx.isin(omit)])
if ((S_new[-1] / S_new[0]) <= stol):
print(
"Whoops! SVD subset selection failed, trying dqrdc2 on full matrix"
)
if (len(X) == 3):
Q = np.matmul(S[2], S[1] + S[0].T)
else:
Q, R, pivot = qr(X, pivoting=True)
pivot = Q[2]
idx = np.argsort(pivot[:k])
omit = pivot[-(nvars - k):]
S_new = svd(X[~idx.isin(omit)])
S = S_new
print("omitting column(s) ", omit)
s = np.zeros(nVars)
nobs = y.shape[0]
nVars = S[2].shape[1]
if weights is None:
weights = np.ones(nobs)
varianceFam = family.variance
linkinvFam = link.inverse
mu_eta = link.inverse_deriv
etastart = None
if len(y.shape) == 1:
mustart = (weights * y + 0.5) / (weights + 1)
else:
n = y + weights
ytmp = 0 if not n else y[:, 1] / n
mustart = (n * ytmp + 0.5) / (n + 1)
eta = link(mustart)
dev_resids = lambda y, m, w: family.resid_dev(y, m, w)**2
dev = sum(dev_resids(y, linkinvFam(eta), weights))
devold = 0
for j in range(maxit):
g = linkinvFam(eta)
varg = varianceFam(g)
if (np.any(np.isnan(varg))):
raise LinAlgError("NAs in variance of the inverse link function")
if (np.any(varg == 0)):
raise LinAlgError(
"Zero value in variance of the inverse link function")
gprime = mu_eta(eta)
if (np.any(np.isnan(gprime))):
raise LinAlgError("NAs in the inverse link function derivative")
z = eta + (y - g) / gprime
W = weights * (gprime**2 / varg)
W = W.reshape(len(W), 1)
cross1 = np.matmul(np.transpose(S[0][:, :7]), W * S[0][:, :7])
C, rank_bn, piv = cholesky_pivot(cross1, full_pivot=True)
cross2 = np.matmul(np.transpose(np.asarray(S[0][:, :7])),
np.asarray(W.reshape(-1) * z))[piv - 1]
s = solve_triangular(np.transpose(C), cross2, lower=True)
s = solve_triangular(C, s, lower=False)[np.argsort(piv)]
eta = np.matmul(S[0][:, :7], s)
dev = np.sum(dev_resids(y, g, weights))
if (np.absolute(dev - devold) / (0.1 + np.absolute(dev)) < tol): break
devold = dev
x = np.empty(X.shape[1])
x[:] = np.nan
inV = 1 / S[1]
if (reg_method == "minimum norm"): inV[inV > 1 / stol] = 1
x[idx] = np.matmul(S[2].T, (s * inV).reshape(-1, 1)).reshape(-1)
x = pd.DataFrame(x, list(X))
return (x, j + 1, k, pivot
) # coefficients=x,iterations=j, rank=k, pivot=pivot
def __init__(self, link=L.logit): # , n=1.):
# TODO: it *should* work for a constant n>1 actually, if data_weights
# is equal to n
self.family = Binomial(link=link)
class BinomialWrapper(FamilyWrapper):
"""
The wrapper of Binomial exponential family distribution,
with function for per sample probability.
Parameters
----------
link : a link instance, optional
The default link for the Binomial family is the logit link.
Available links are logit, probit, cauchy, log, and cloglog.
See statsmodels.family.links for more information.
Attributes
----------
family : a statsmodel Binomial family object
--------
"""
def __init__(self, link=L.logit): # , n=1.):
# TODO: it *should* work for a constant n>1 actually, if data_weights
# is equal to n
self.family = Binomial(link=link)
def loglike_per_sample(self, endog, mu, scale=1.):
"""
The function to calculate log-likelihood per sample
in terms of the fitted mean response.
Parameters
----------
endog : array-like of shape (n, k) or (n, )
Endogenous response variable
mu : array-like of shape (n, )
Fitted mean response variable
scale : float, optional
Not used for the Binomial GLM.
Returns
-------
log_p : array-like of shape (n, )
The value of the loglikelihood function evaluated per sample
(endog,mu,freq_weights,scale) as defined below.
Notes
--------
If the endogenous variable is binary:
.. math::
log_p_{i} = (y_i * \log(\mu_i/(1-\mu_i)) + \log(1-\mu_i))
If the endogenous variable is binomial:
.. math::
log_p_{i} = (\ln \Gamma(n+1) -
\ln \Gamma(y_i + 1) - \ln \Gamma(n_i - y_i +1) + y_i *
\log(\mu_i / (n_i - \mu_i)) + n * \log(1 - \mu_i/n_i))
where :math:`y_i = Y_i * n_i` with :math:`Y_i` and :math:`n_i` as
defined in Binomial initialize. This simply makes :math:`y_i` the
original number of successes.
"""
# special setup
# see _Setup_binomial(self) in generalized_linear_model.py
tmp = self.family.initialize(endog, 1)
endog = tmp[0]
if np.shape(self.family.n) == () and self.family.n == 1:
return scale * (endog * np.log(old_div(mu, (1 - mu)) + 1e-200) +
np.log(1 - mu)).reshape(-1,)
else:
y = endog * self.family.n # convert back to successes
return scale * (special.gammaln(self.family.n + 1) -
special.gammaln(y + 1) -
special.gammaln(self.family.n - y + 1) + y *
np.log(old_div(mu, (1 - mu))) + self.family.n *
np.log(1 - mu)).reshape(-1,)