def p(x_new, reg=reg_choice): y = ohie_0m.ohp_all_ever_admin x = ohie_0m.hhinc_pctfpl_0m.values.reshape((-1, 1)) if reg == 'linear': model = LinearRegression().fit(x, y) return model.predict(x_new) elif reg == 'probit': probit_model = Probit(y, x).fit() return probit_model.predict(x_new)
def eta0(x_new, reg=reg_choice): y = ohie_0m_d0.health_gen_bin_0m x = ohie_0m_d0.hhinc_pctfpl_0m.values.reshape((-1, 1)) if reg == 'linear': model = LinearRegression().fit(x, y) return model.predict(x_new) elif reg == 'probit': probit_model = Probit(y, x).fit() return probit_model.predict(x_new)
class GenericZeroInflated(CountModel): __doc__ = """ Generiz Zero Inflated model for count data %(params)s %(extra_params)s Attributes ----------- endog : array A reference to the endogenous response variable exog : array A reference to the exogenous design. exog_infl: array A reference to the zero-inflated exogenous design. """ % {'params' : base._model_params_doc, 'extra_params' : _doc_zi_params + base._missing_param_doc} def __init__(self, endog, exog, exog_infl=None, offset=None, inflation='logit', exposure=None, missing='none', **kwargs): super(GenericZeroInflated, self).__init__(endog, exog, offset=offset, exposure=exposure, missing=missing, **kwargs) if exog_infl is None: self.k_inflate = 1 self.exog_infl = np.ones((endog.size, self.k_inflate), dtype=np.float64) else: self.exog_infl = exog_infl self.k_inflate = exog_infl.shape[1] if len(exog.shape) == 1: self.k_exog = 1 else: self.k_exog = exog.shape[1] self.infl = inflation if inflation == 'logit': self.model_infl = Logit(np.zeros(self.exog_infl.shape[0]), self.exog_infl) self._hessian_inflate = self._hessian_logit elif inflation == 'probit': self.model_infl = Probit(np.zeros(self.exog_infl.shape[0]), self.exog_infl) self._hessian_inflate = self._hessian_probit else: raise TypeError("inflation == %s, which is not handled" % inflation) self.inflation = inflation self.k_extra = self.k_inflate if len(self.exog) != len(self.exog_infl): raise ValueError('exog and exog_infl have different number of' 'observation. `missing` handling is not supported') infl_names = ['inflate_%s' % i for i in self.model_infl.data.param_names] self.exog_names[:] = infl_names + list(self.exog_names) self.exog_infl = np.asarray(self.exog_infl, dtype=np.float64) self._init_keys.extend(['exog_infl', 'inflation']) self._null_drop_keys = ['exog_infl'] def loglike(self, params): """ Loglikelihood of Generic Zero Inflated model Parameters ---------- params : array-like The parameters of the model. Returns ------- loglike : float The log-likelihood function of the model evaluated at `params`. See notes. Notes -------- .. math:: \\ln L=\\sum_{y_{i}=0}\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+ \\sum_{y_{i}>0}(\\ln(1-w_{i})+L_{main\\_model}) where P - pdf of main model, L - loglike function of main model. """ return np.sum(self.loglikeobs(params)) def loglikeobs(self, params): """ Loglikelihood for observations of Generic Zero Inflated model Parameters ---------- params : array-like The parameters of the model. Returns ------- loglike : ndarray The log likelihood for each observation of the model evaluated at `params`. See Notes Notes -------- .. math:: \\ln L=\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+ \\ln(1-w_{i})+L_{main\\_model} where P - pdf of main model, L - loglike function of main model. for observations :math:`i=1,...,n` """ params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) llf_main = self.model_main.loglikeobs(params_main) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] llf = np.zeros_like(y, dtype=np.float64) llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx]))) llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx] return llf def fit(self, start_params=None, method='bfgs', maxiter=35, full_output=1, disp=1, callback=None, cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs): if start_params is None: offset = getattr(self, "offset", 0) + getattr(self, "exposure", 0) if np.size(offset) == 1 and offset == 0: offset = None start_params = self._get_start_params() if callback is None: # work around perfect separation callback #3895 callback = lambda *x: x mlefit = super(GenericZeroInflated, self).fit(start_params=start_params, maxiter=maxiter, disp=disp, method=method, full_output=full_output, callback=callback, **kwargs) zipfit = self.result_class(self, mlefit._results) result = self.result_class_wrapper(zipfit) if cov_kwds is None: cov_kwds = {} result._get_robustcov_results(cov_type=cov_type, use_self=True, use_t=use_t, **cov_kwds) return result fit.__doc__ = DiscreteModel.fit.__doc__ def fit_regularized(self, start_params=None, method='l1', maxiter='defined_by_method', full_output=1, disp=1, callback=None, alpha=0, trim_mode='auto', auto_trim_tol=0.01, size_trim_tol=1e-4, qc_tol=0.03, **kwargs): if np.size(alpha) == 1 and alpha != 0: k_params = self.k_exog + self.k_inflate alpha = alpha * np.ones(k_params) extra = self.k_extra - self.k_inflate alpha_p = alpha[:-(self.k_extra - extra)] if (self.k_extra and np.size(alpha) > 1) else alpha if start_params is None: offset = getattr(self, "offset", 0) + getattr(self, "exposure", 0) if np.size(offset) == 1 and offset == 0: offset = None start_params = self.model_main.fit_regularized( start_params=start_params, method=method, maxiter=maxiter, full_output=full_output, disp=0, callback=callback, alpha=alpha_p, trim_mode=trim_mode, auto_trim_tol=auto_trim_tol, size_trim_tol=size_trim_tol, qc_tol=qc_tol, **kwargs).params start_params = np.append(np.ones(self.k_inflate), start_params) cntfit = super(CountModel, self).fit_regularized( start_params=start_params, method=method, maxiter=maxiter, full_output=full_output, disp=disp, callback=callback, alpha=alpha, trim_mode=trim_mode, auto_trim_tol=auto_trim_tol, size_trim_tol=size_trim_tol, qc_tol=qc_tol, **kwargs) if method in ['l1', 'l1_cvxopt_cp']: discretefit = self.result_class_reg(self, cntfit) else: raise TypeError( "argument method == %s, which is not handled" % method) return self.result_class_reg_wrapper(discretefit) fit_regularized.__doc__ = DiscreteModel.fit_regularized.__doc__ def score_obs(self, params): """ Generic Zero Inflated model score (gradient) vector of the log-likelihood Parameters ---------- params : array-like The parameters of the model Returns ------- score : ndarray, 1-D The score vector of the model, i.e. the first derivative of the loglikelihood function, evaluated at `params` """ params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) score_main = self.model_main.score_obs(params_main) llf_main = self.model_main.loglikeobs(params_main) llf = self.loglikeobs(params) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] mu = self.model_main.predict(params_main) dldp = np.zeros((self.exog.shape[0], self.k_exog), dtype=np.float64) dldw = np.zeros_like(self.exog_infl, dtype=np.float64) dldp[zero_idx,:] = (score_main[zero_idx].T * (1 - (w[zero_idx]) / np.exp(llf[zero_idx]))).T dldp[nonzero_idx,:] = score_main[nonzero_idx] if self.inflation == 'logit': dldw[zero_idx,:] = (self.exog_infl[zero_idx].T * w[zero_idx] * (1 - w[zero_idx]) * (1 - np.exp(llf_main[zero_idx])) / np.exp(llf[zero_idx])).T dldw[nonzero_idx,:] = -(self.exog_infl[nonzero_idx].T * w[nonzero_idx]).T elif self.inflation == 'probit': return approx_fprime(params, self.loglikeobs) return np.hstack((dldw, dldp)) def score(self, params): return self.score_obs(params).sum(0) def _hessian_main(self, params): pass def _hessian_logit(self, params): params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) score_main = self.model_main.score_obs(params_main) llf_main = self.model_main.loglikeobs(params_main) llf = self.loglikeobs(params) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] hess_arr = np.zeros((self.k_inflate, self.k_exog + self.k_inflate)) pmf = np.exp(llf) #d2l/dw2 for i in range(self.k_inflate): for j in range(i, -1, -1): hess_arr[i, j] = (( self.exog_infl[zero_idx, i] * self.exog_infl[zero_idx, j] * (w[zero_idx] * (1 - w[zero_idx]) * ((1 - np.exp(llf_main[zero_idx])) * (1 - 2 * w[zero_idx]) * np.exp(llf[zero_idx]) - (w[zero_idx] - w[zero_idx]**2) * (1 - np.exp(llf_main[zero_idx]))**2) / pmf[zero_idx]**2)).sum() - (self.exog_infl[nonzero_idx, i] * self.exog_infl[nonzero_idx, j] * w[nonzero_idx] * (1 - w[nonzero_idx])).sum()) #d2l/dpdw for i in range(self.k_inflate): for j in range(self.k_exog): hess_arr[i, j + self.k_inflate] = -(score_main[zero_idx, j] * w[zero_idx] * (1 - w[zero_idx]) * self.exog_infl[zero_idx, i] / pmf[zero_idx]).sum() return hess_arr def _hessian_probit(self, params): pass def hessian(self, params): """ Generic Zero Inflated model Hessian matrix of the loglikelihood Parameters ---------- params : array-like The parameters of the model Returns ------- hess : ndarray, (k_vars, k_vars) The Hessian, second derivative of loglikelihood function, evaluated at `params` Notes ----- """ hess_arr_main = self._hessian_main(params) hess_arr_infl = self._hessian_inflate(params) if hess_arr_main is None or hess_arr_infl is None: return approx_hess(params, self.loglike) dim = self.k_exog + self.k_inflate hess_arr = np.zeros((dim, dim)) hess_arr[:self.k_inflate,:] = hess_arr_infl hess_arr[self.k_inflate:,self.k_inflate:] = hess_arr_main tri_idx = np.triu_indices(self.k_exog + self.k_inflate, k=1) hess_arr[tri_idx] = hess_arr.T[tri_idx] return hess_arr def predict(self, params, exog=None, exog_infl=None, exposure=None, offset=None, which='mean'): """ Predict response variable of a count model given exogenous variables. Parameters ---------- params : array-like The parameters of the model exog : array, optional A reference to the exogenous design. If not assigned, will be used exog from fitting. exog_infl : array, optional A reference to the zero-inflated exogenous design. If not assigned, will be used exog from fitting. offset : array, optional Offset is added to the linear prediction with coefficient equal to 1. exposure : array, optional Log(exposure) is added to the linear prediction with coefficient equal to 1. If exposure is specified, then it will be logged by the method. The user does not need to log it first. which : string, optional Define values that will be predicted. 'mean', 'mean-main', 'linear', 'mean-nonzero', 'prob-zero, 'prob', 'prob-main' Default is 'mean'. Notes ----- """ if exog is None: exog = self.exog if exog_infl is None: exog_infl = self.exog_infl if exposure is None: exposure = getattr(self, 'exposure', 0) else: exposure = np.log(exposure) if offset is None: offset = 0 params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] prob_main = 1 - self.model_infl.predict(params_infl, exog_infl) lin_pred = np.dot(exog, params_main[:self.exog.shape[1]]) + exposure + offset # Refactor: This is pretty hacky, # there should be an appropriate predict method in model_main # this is just prob(y=0 | model_main) tmp_exog = self.model_main.exog tmp_endog = self.model_main.endog tmp_offset = getattr(self.model_main, 'offset', ['no']) tmp_exposure = getattr(self.model_main, 'exposure', ['no']) self.model_main.exog = exog self.model_main.endog = np.zeros((exog.shape[0])) self.model_main.offset = offset self.model_main.exposure = exposure llf = self.model_main.loglikeobs(params_main) self.model_main.exog = tmp_exog self.model_main.endog = tmp_endog # tmp_offset might be an array with elementwise equality testing if len(tmp_offset) == 1 and tmp_offset[0] == 'no': del self.model_main.offset else: self.model_main.offset = tmp_offset if len(tmp_exposure) == 1 and tmp_exposure[0] == 'no': del self.model_main.exposure else: self.model_main.exposure = tmp_exposure # end hack prob_zero = (1 - prob_main) + prob_main * np.exp(llf) if which == 'mean': return prob_main * np.exp(lin_pred) elif which == 'mean-main': return np.exp(lin_pred) elif which == 'linear': return lin_pred elif which == 'mean-nonzero': return prob_main * np.exp(lin_pred) / (1 - prob_zero) elif which == 'prob-zero': return prob_zero elif which == 'prob-main': return prob_main elif which == 'prob': return self._predict_prob(params, exog, exog_infl, exposure, offset) else: raise ValueError('which = %s is not available' % which)
class GenericZeroInflated(CountModel): __doc__ = """ Generic Zero Inflated Model %(params)s %(extra_params)s Attributes ---------- endog : ndarray A reference to the endogenous response variable exog : ndarray A reference to the exogenous design. exog_infl : ndarray A reference to the zero-inflated exogenous design. """ % {'params' : base._model_params_doc, 'extra_params' : _doc_zi_params + base._missing_param_doc} def __init__(self, endog, exog, exog_infl=None, offset=None, inflation='logit', exposure=None, missing='none', **kwargs): super(GenericZeroInflated, self).__init__(endog, exog, offset=offset, exposure=exposure, missing=missing, **kwargs) if exog_infl is None: self.k_inflate = 1 self._no_exog_infl = True self.exog_infl = np.ones((endog.size, self.k_inflate), dtype=np.float64) else: self.exog_infl = exog_infl self.k_inflate = exog_infl.shape[1] self._no_exog_infl = False if len(exog.shape) == 1: self.k_exog = 1 else: self.k_exog = exog.shape[1] self.infl = inflation if inflation == 'logit': self.model_infl = Logit(np.zeros(self.exog_infl.shape[0]), self.exog_infl) self._hessian_inflate = self._hessian_logit elif inflation == 'probit': self.model_infl = Probit(np.zeros(self.exog_infl.shape[0]), self.exog_infl) self._hessian_inflate = self._hessian_probit else: raise ValueError("inflation == %s, which is not handled" % inflation) self.inflation = inflation self.k_extra = self.k_inflate if len(self.exog) != len(self.exog_infl): raise ValueError('exog and exog_infl have different number of' 'observation. `missing` handling is not supported') infl_names = ['inflate_%s' % i for i in self.model_infl.data.param_names] self.exog_names[:] = infl_names + list(self.exog_names) self.exog_infl = np.asarray(self.exog_infl, dtype=np.float64) self._init_keys.extend(['exog_infl', 'inflation']) self._null_drop_keys = ['exog_infl'] def _get_exogs(self): """list of exogs, for internal use in post-estimation """ return (self.exog, self.exog_infl) def loglike(self, params): """ Loglikelihood of Generic Zero Inflated model. Parameters ---------- params : array_like The parameters of the model. Returns ------- loglike : float The log-likelihood function of the model evaluated at `params`. See notes. Notes -------- .. math:: \\ln L=\\sum_{y_{i}=0}\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+ \\sum_{y_{i}>0}(\\ln(1-w_{i})+L_{main\\_model}) where P - pdf of main model, L - loglike function of main model. """ return np.sum(self.loglikeobs(params)) def loglikeobs(self, params): """ Loglikelihood for observations of Generic Zero Inflated model. Parameters ---------- params : array_like The parameters of the model. Returns ------- loglike : ndarray The log likelihood for each observation of the model evaluated at `params`. See Notes for definition. Notes -------- .. math:: \\ln L=\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+ \\ln(1-w_{i})+L_{main\\_model} where P - pdf of main model, L - loglike function of main model. for observations :math:`i=1,...,n` """ params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) llf_main = self.model_main.loglikeobs(params_main) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] llf = np.zeros_like(y, dtype=np.float64) llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx]))) llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx] return llf @Appender(DiscreteModel.fit.__doc__) def fit(self, start_params=None, method='bfgs', maxiter=35, full_output=1, disp=1, callback=None, cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs): if start_params is None: offset = getattr(self, "offset", 0) + getattr(self, "exposure", 0) if np.size(offset) == 1 and offset == 0: offset = None start_params = self._get_start_params() if callback is None: # work around perfect separation callback #3895 callback = lambda *x: x mlefit = super(GenericZeroInflated, self).fit(start_params=start_params, maxiter=maxiter, disp=disp, method=method, full_output=full_output, callback=callback, **kwargs) zipfit = self.result_class(self, mlefit._results) result = self.result_class_wrapper(zipfit) if cov_kwds is None: cov_kwds = {} result._get_robustcov_results(cov_type=cov_type, use_self=True, use_t=use_t, **cov_kwds) return result @Appender(DiscreteModel.fit_regularized.__doc__) def fit_regularized(self, start_params=None, method='l1', maxiter='defined_by_method', full_output=1, disp=1, callback=None, alpha=0, trim_mode='auto', auto_trim_tol=0.01, size_trim_tol=1e-4, qc_tol=0.03, **kwargs): _validate_l1_method(method) if np.size(alpha) == 1 and alpha != 0: k_params = self.k_exog + self.k_inflate alpha = alpha * np.ones(k_params) extra = self.k_extra - self.k_inflate alpha_p = alpha[:-(self.k_extra - extra)] if (self.k_extra and np.size(alpha) > 1) else alpha if start_params is None: offset = getattr(self, "offset", 0) + getattr(self, "exposure", 0) if np.size(offset) == 1 and offset == 0: offset = None start_params = self.model_main.fit_regularized( start_params=start_params, method=method, maxiter=maxiter, full_output=full_output, disp=0, callback=callback, alpha=alpha_p, trim_mode=trim_mode, auto_trim_tol=auto_trim_tol, size_trim_tol=size_trim_tol, qc_tol=qc_tol, **kwargs).params start_params = np.append(np.ones(self.k_inflate), start_params) cntfit = super(CountModel, self).fit_regularized( start_params=start_params, method=method, maxiter=maxiter, full_output=full_output, disp=disp, callback=callback, alpha=alpha, trim_mode=trim_mode, auto_trim_tol=auto_trim_tol, size_trim_tol=size_trim_tol, qc_tol=qc_tol, **kwargs) discretefit = self.result_class_reg(self, cntfit) return self.result_class_reg_wrapper(discretefit) def score_obs(self, params): """ Generic Zero Inflated model score (gradient) vector of the log-likelihood Parameters ---------- params : array_like The parameters of the model Returns ------- score : ndarray, 1-D The score vector of the model, i.e. the first derivative of the loglikelihood function, evaluated at `params` """ params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) score_main = self.model_main.score_obs(params_main) llf_main = self.model_main.loglikeobs(params_main) llf = self.loglikeobs(params) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] mu = self.model_main.predict(params_main) # TODO: need to allow for complex to use CS numerical derivatives dldp = np.zeros((self.exog.shape[0], self.k_exog), dtype=np.float64) dldw = np.zeros_like(self.exog_infl, dtype=np.float64) dldp[zero_idx,:] = (score_main[zero_idx].T * (1 - (w[zero_idx]) / np.exp(llf[zero_idx]))).T dldp[nonzero_idx,:] = score_main[nonzero_idx] if self.inflation == 'logit': dldw[zero_idx,:] = (self.exog_infl[zero_idx].T * w[zero_idx] * (1 - w[zero_idx]) * (1 - np.exp(llf_main[zero_idx])) / np.exp(llf[zero_idx])).T dldw[nonzero_idx,:] = -(self.exog_infl[nonzero_idx].T * w[nonzero_idx]).T elif self.inflation == 'probit': return approx_fprime(params, self.loglikeobs) return np.hstack((dldw, dldp)) def score(self, params): return self.score_obs(params).sum(0) def _hessian_main(self, params): pass def _hessian_logit(self, params): params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] y = self.endog w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) score_main = self.model_main.score_obs(params_main) llf_main = self.model_main.loglikeobs(params_main) llf = self.loglikeobs(params) zero_idx = np.nonzero(y == 0)[0] nonzero_idx = np.nonzero(y)[0] hess_arr = np.zeros((self.k_inflate, self.k_exog + self.k_inflate)) pmf = np.exp(llf) #d2l/dw2 for i in range(self.k_inflate): for j in range(i, -1, -1): hess_arr[i, j] = (( self.exog_infl[zero_idx, i] * self.exog_infl[zero_idx, j] * (w[zero_idx] * (1 - w[zero_idx]) * ((1 - np.exp(llf_main[zero_idx])) * (1 - 2 * w[zero_idx]) * np.exp(llf[zero_idx]) - (w[zero_idx] - w[zero_idx]**2) * (1 - np.exp(llf_main[zero_idx]))**2) / pmf[zero_idx]**2)).sum() - (self.exog_infl[nonzero_idx, i] * self.exog_infl[nonzero_idx, j] * w[nonzero_idx] * (1 - w[nonzero_idx])).sum()) #d2l/dpdw for i in range(self.k_inflate): for j in range(self.k_exog): hess_arr[i, j + self.k_inflate] = -(score_main[zero_idx, j] * w[zero_idx] * (1 - w[zero_idx]) * self.exog_infl[zero_idx, i] / pmf[zero_idx]).sum() return hess_arr def _hessian_probit(self, params): pass def hessian(self, params): """ Generic Zero Inflated model Hessian matrix of the loglikelihood Parameters ---------- params : array_like The parameters of the model Returns ------- hess : ndarray, (k_vars, k_vars) The Hessian, second derivative of loglikelihood function, evaluated at `params` Notes ----- """ hess_arr_main = self._hessian_main(params) hess_arr_infl = self._hessian_inflate(params) if hess_arr_main is None or hess_arr_infl is None: return approx_hess(params, self.loglike) dim = self.k_exog + self.k_inflate hess_arr = np.zeros((dim, dim)) hess_arr[:self.k_inflate,:] = hess_arr_infl hess_arr[self.k_inflate:,self.k_inflate:] = hess_arr_main tri_idx = np.triu_indices(self.k_exog + self.k_inflate, k=1) hess_arr[tri_idx] = hess_arr.T[tri_idx] return hess_arr def predict(self, params, exog=None, exog_infl=None, exposure=None, offset=None, which='mean', y_values=None): """ Predict response variable or other statistic given exogenous variables. Parameters ---------- params : array_like The parameters of the model. exog : ndarray, optional Explanatory variables for the main count model. If ``exog`` is None, then the data from the model will be used. exog_infl : ndarray, optional Explanatory variables for the zero-inflation model. ``exog_infl`` has to be provided if ``exog`` was provided unless ``exog_infl`` in the model is only a constant. offset : ndarray, optional Offset is added to the linear predictor of the mean function with coefficient equal to 1. Default is zero if exog is not None, and the model offset if exog is None. exposure : ndarray, optional Log(exposure) is added to the linear predictor with coefficient equal to 1. If exposure is specified, then it will be logged by the method. The user does not need to log it first. Default is one if exog is is not None, and it is the model exposure if exog is None. which : str (optional) Statitistic to predict. Default is 'mean'. - 'mean' : the conditional expectation of endog E(y | x), i.e. exp of linear predictor. - 'linear' : the linear predictor of the mean function. - 'var' : returns the estimated variance of endog implied by the model. - 'mean-main' : mean of the main count model - 'prob-main' : probability of selecting the main model. The probability of zero inflation is ``1 - prob-main``. - 'mean-nonzero' : expected value conditional on having observation larger than zero, E(y | X, y>0) - 'prob-zero' : probability of observing a zero count. P(y=0 | x) - 'prob' : probabilities of each count from 0 to max(endog), or for y_values if those are provided. This is a multivariate return (2-dim when predicting for several observations). y_values : array_like Values of the random variable endog at which pmf is evaluated. Only used if ``which="prob"`` """ no_exog = False if exog is None: no_exog = True exog = self.exog if exog_infl is None: if no_exog: exog_infl = self.exog_infl else: if self._no_exog_infl: exog_infl = np.ones((len(exog), 1)) else: exog_infl = np.asarray(exog_infl) if exog_infl.ndim == 1 and self.k_inflate == 1: exog_infl = exog_infl[:, None] if exposure is None: if no_exog: exposure = getattr(self, 'exposure', 0) else: exposure = 0 else: exposure = np.log(exposure) if offset is None: if no_exog: offset = getattr(self, 'offset', 0) else: offset = 0 params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] prob_main = 1 - self.model_infl.predict(params_infl, exog_infl) lin_pred = np.dot(exog, params_main[:self.exog.shape[1]]) + exposure + offset # Refactor: This is pretty hacky, # there should be an appropriate predict method in model_main # this is just prob(y=0 | model_main) tmp_exog = self.model_main.exog tmp_endog = self.model_main.endog tmp_offset = getattr(self.model_main, 'offset', False) tmp_exposure = getattr(self.model_main, 'exposure', False) self.model_main.exog = exog self.model_main.endog = np.zeros((exog.shape[0])) self.model_main.offset = offset self.model_main.exposure = exposure llf = self.model_main.loglikeobs(params_main) self.model_main.exog = tmp_exog self.model_main.endog = tmp_endog # tmp_offset might be an array with elementwise equality testing #if np.size(tmp_offset) == 1 and tmp_offset[0] == 'no': if tmp_offset is False: del self.model_main.offset else: self.model_main.offset = tmp_offset #if np.size(tmp_exposure) == 1 and tmp_exposure[0] == 'no': if tmp_exposure is False: del self.model_main.exposure else: self.model_main.exposure = tmp_exposure # end hack prob_zero = (1 - prob_main) + prob_main * np.exp(llf) if which == 'mean': return prob_main * np.exp(lin_pred) elif which == 'mean-main': return np.exp(lin_pred) elif which == 'linear': return lin_pred elif which == 'mean-nonzero': return prob_main * np.exp(lin_pred) / (1 - prob_zero) elif which == 'prob-zero': return prob_zero elif which == 'prob-main': return prob_main elif which == 'var': mu = np.exp(lin_pred) return self._predict_var(params, mu, 1 - prob_main) elif which == 'prob': return self._predict_prob(params, exog, exog_infl, exposure, offset, y_values=y_values) else: raise ValueError('which = %s is not available' % which) def _derivative_predict(self, params, exog=None, transform='dydx'): """NotImplemented """ raise NotImplementedError def _derivative_exog(self, params, exog=None, transform="dydx", dummy_idx=None, count_idx=None): """NotImplemented """ raise NotImplementedError def _deriv_mean_dparams(self, params): """ Derivative of the expected endog with respect to the parameters. Parameters ---------- params : ndarray parameter at which score is evaluated Returns ------- The value of the derivative of the expected endog with respect to the parameter vector. """ params_infl = params[:self.k_inflate] params_main = params[self.k_inflate:] w = self.model_infl.predict(params_infl) w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps) mu = self.model_main.predict(params_main) score_infl = self.model_infl._deriv_mean_dparams(params_infl) score_main = self.model_main._deriv_mean_dparams(params_main) dmat_infl = - mu[:, None] * score_infl dmat_main = (1 - w[:, None]) * score_main dmat = np.column_stack((dmat_infl, dmat_main)) return dmat def _deriv_score_obs_dendog(self, params): """derivative of score_obs w.r.t. endog Parameters ---------- params : ndarray parameter at which score is evaluated Returns ------- derivative : ndarray_2d The derivative of the score_obs with respect to endog. """ raise NotImplementedError # The below currently does not work, discontinuity at zero # see https://github.com/statsmodels/statsmodels/pull/7951#issuecomment-996355875 # noqa from statsmodels.tools.numdiff import _approx_fprime_scalar endog_original = self.endog def f(y): if y.ndim == 2 and y.shape[1] == 1: y = y[:, 0] self.endog = y self.model_main.endog = y sf = self.score_obs(params) self.endog = endog_original self.model_main.endog = endog_original return sf ds = _approx_fprime_scalar(self.endog[:, None], f, epsilon=1e-2) return ds
def flip_bits(y, p): x = np.random.rand(y.shape[0], 1) < p y[x < p] = 1 - y[x < p] return y n, d = 100, 2 data_x = np.random.randn(n, d) w = np.random.randn(d, 1) data_y = flip_bits((data_x @ w > 0), 0) lam = 1e-2 # statsmodel.Probit sm_probit_reg = Probit(exog=data_x, endog=data_y).fit(disp=0, method='bfgs') sm_probit_prob = sm_probit_reg.predict(exog=data_x) # Our Implementation: probit_reg = ProbitReg() # EM: em_w, obj_trace_em = probit_reg.probreg_fit_em(data_x, data_y, lam) em_ypred, em_prob = probit_reg.predict(data_x, em_w) # gradient: gradient_w, obj_trace_gradient = probit_reg.probit_reg_fit_gradient( data_x, data_y, lam) gradient_ypred, gradient_prob = probit_reg.predict(data_x, gradient_w) plt.figure() plt.plot(sm_probit_prob, em_prob, 'o')
class ProbitRegression(Learner): """ The probit regression learning algorithm. Given data, this class constructs and stores a probability unit regression mdl that can be used to quantify the probability of testing data-points taking on certain class values. """ def __init__(self, alpha: float, **params: any): """ Initialises the Probit regression algorithm. :param alpha: regularization term alpha. :param params: Ignored. """ super().__init__(**params) self.name = 'Probit Regression' self.alpha = alpha self.gamma = 0.5 self.add_intercept = True self.binary_points = True self.beta = list() self.data: Optional[RecordSet] = None self.model: Optional[Probit] = None # will be set during fit def fit(self, rs: RecordSet) -> None: """ fit a Probit regression mdl :param rs: The record set to fit with. """ # set params self.data = cp.deepcopy(rs) patterns = self.data.entries[:, :-1] out = self.data.entries[:, -1:] if self.add_intercept: intercept = np.ones((patterns.shape[0], 1)) patterns = np.hstack((intercept, patterns)) # avoid error if self.alpha == 0: raise Exception("Alpha Probit too low to obtain reliable results") self.model = Probit(endog=out.ravel(), exog=patterns) self.model = self.model.fit_regularized(alpha=self.alpha, maxiter=10e5, disp=False) def predict(self, rs: RecordSet) -> np.ndarray: """ Assigns a predicted class label to the given record sets. :param rs: The record set to assign predictions to. :return: A column vector of predictions corresponding to the record set's rows. """ # set params patterns = rs.entries[:, :-1] if self.add_intercept: intercept = np.ones((patterns.shape[0], 1)) patterns = np.hstack((intercept, patterns)) # predict predictions = self.model.predict(exog=patterns) if self.binary_points: predictions = self.discrete_points(predictions=predictions) # return 2d predictions = np.reshape(predictions, (-1, 1)) return predictions def discrete_points(self, predictions): """ Turns probabilities into discrete classes :param predictions: The predicted class probabilities :return: A vector with discrete classes """ n = predictions.shape[0] for i in range(0, n): if predictions[i] >= self.gamma: predictions[i] = 1 else: predictions[i] = 0 return predictions