def setupClass(cls): data = sm.datasets.anes96.load() cls.data = data exog = data.exog exog = sm.add_constant(exog, prepend=False) mymodel = MNLogit(data.endog, exog) cls.res1 = mymodel.fit(method="lbfgs", disp=0, maxiter=50000, #m=12, pgtol=1e-7, factr=1e3, # 5 failures #m=20, pgtol=1e-8, factr=1e2, # 3 failures #m=30, pgtol=1e-9, factr=1e1, # 1 failure m=40, pgtol=1e-10, factr=5e0, loglike_and_score=mymodel.loglike_and_score) res2 = Anes() res2.mnlogit_basezero() cls.res2 = res2
def setupClass(cls): data = sm.datasets.anes96.load() cls.data = data exog = data.exog exog = sm.add_constant(exog, prepend=False) mymodel = MNLogit(data.endog, exog) cls.res1 = mymodel.fit(method="lbfgs", disp=0, maxiter=50000, #m=12, pgtol=1e-7, factr=1e3, # 5 failures #m=20, pgtol=1e-8, factr=1e2, # 3 failures #m=30, pgtol=1e-9, factr=1e1, # 1 failure m=40, pgtol=1e-10, factr=5e0, loglike_and_score=mymodel.loglike_and_score) res2 = Anes() res2.mnlogit_basezero() cls.res2 = res2
Note IRis data has perfect separation problem. add some noise to it!!! ''' iris = datasets.load_iris() x = iris.data # lets use the first two features!! x = np.c_[np.ones(x.shape[0]), x] print(x.shape) print(x[:4]) y = iris.target y[1:5] = 2 y[-5:] = 1 xTrain, xTest, yTrain, yTest = train_test_split( x, y, test_size=0.33, random_state=50) # equal size splits! print(xTest.shape) logitModel = MNLogit(yTrain, xTrain) rs = logitModel.fit(maxiter=200) print(rs.params) print(rs.summary()) yPred = rs.predict() print(yPred, yPred.shape, '\n\n') def probtolabelPred(yPred): print(yPred[:10]) labs = np.argmax(yPred, axis=1) return labs predLabs = probtolabelPred(yPred) print(predLabs)
def multicategorical_continuous(dependent_var: pd.Series, independent_var: pd.Series, list_columns_names_export: list): dependent_var_name = dependent_var.name independent_var_name = independent_var.name long_data = pd.DataFrame.from_dict({"dependent_var_name": dependent_var, "independent_var_name": independent_var}, orient="columns") groupby_stats = long_data \ .groupby("dependent_var_name") \ .agg(["min", "max", "median", "mean", "count"]) \ .droplevel(level=0, axis=1) \ .rename({"count": "nonNA_count"}, axis=1) \ .reset_index(drop=False) \ .melt(id_vars="dependent_var_name", var_name="indicator") \ .rename({"dependent_var_name": "dependent_var_modality"}, axis=1)\ .round({"value": 2}) ############### ############# ANOVA from scipy.stats import f_oneway wide_data = [col.dropna() for _, col in long_data.pivot(index=None, columns="dependent_var_name", values="independent_var_name").iteritems()] anova = pd.DataFrame.from_dict({ "dependent_var_modality": "overall_margin", "indicator": "anova_oneway", "value": f_oneway(*wide_data).pvalue}, orient="index", ).transpose() ############# ########## MODEL dependent_var_cat = dependent_var.astype(CategoricalDtype(ordered=False)) ref_modality_dependent = groupby_stats.loc[lambda df: df["indicator"] == "nonNA_count", :] \ .loc[lambda df: df["value"] == df["value"].max(), :] \ .iloc[0, :] \ ["dependent_var_modality"] new_levels = [ref_modality_dependent] + pd.CategoricalIndex(dependent_var_cat) \ .remove_categories(ref_modality_dependent).categories.tolist() dependent_var_cat.cat.reorder_categories(new_levels, inplace=True) X = independent_var.rename(independent_var_name).to_frame().assign(intercept=1) model = MNLogit(dependent_var_cat, X) results = model.fit() params = results.params params.columns = dependent_var_cat.cat.categories[1:] params = params.rename_axis("dependent_var_modality", axis=1) \ .rename_axis("independent_var", axis=0) \ .drop("intercept", axis=0) \ .melt(var_name="dependent_var_modality") \ .assign(indicator="coeffs_LogR") ########### LRT LRT = pd.DataFrame.from_dict({ "dependent_var_modality": "overall_margin", "indicator": "pvalue_LRT_LogR", "value": results.llr_pvalue }, orient="index").transpose() ########## pvalues model pvalues = results.pvalues pvalues.columns = dependent_var_cat.cat.categories[1:] pvalues = pvalues.rename_axis("independent_var_name") \ .drop("intercept", axis=0) \ .reset_index(drop=False) \ .melt(id_vars="independent_var_name", var_name="dependent_var_modality") \ .assign(indicator="pvalue_coeff_LogR") \ .drop("independent_var_name", axis=1) ####### conf int param model conf_int = results.conf_int() \ .reset_index(level=1, drop=False) \ .rename({"level_1": "independent_var_name"}, axis=1) \ .rename_axis("dependent_var_modality", axis=0) \ .loc[lambda df: df["independent_var_name"] != "intercept", :] \ .reset_index(drop=False) \ .rename({"lower": "coeff_LogR_lb", "upper": "coeff_LogR_ub"}, axis=1) \ .melt(id_vars=["dependent_var_modality", "independent_var_name"], value_vars=["coeff_LogR_lb", "coeff_LogR_ub"], var_name="indicator") \ .drop("independent_var_name", axis=1) multicategorical_continuous = pd.concat([groupby_stats, params, pvalues, conf_int, LRT, anova], axis=0) \ .assign(ref_modality_dependent=ref_modality_dependent, ref_modality_independent=np.NaN, independent_var_modality=np.NaN, independent_var_name=independent_var_name, dependent_var_name=dependent_var_name) return multicategorical_continuous[list_columns_names_export]
def multicategorical_multicategorical(dependent_var:pd.Series, independent_var:pd.Series, list_columns_names_export: list): dependent_var_name = dependent_var.name independent_var_name = independent_var.name crosscount = pd.crosstab(index=dependent_var, columns=independent_var, margins=True, margins_name="overall_margin") ref_modality_independent = crosscount.transpose() \ .loc[ lambda df: df["overall_margin"] == df["overall_margin"].drop("overall_margin").max(), :] \ .index[0] independent_dummies = pd.get_dummies(independent_var, drop_first=False) \ .drop(ref_modality_independent, axis=1) \ .assign(intercept=1) dependent_var_cat = dependent_var.astype(CategoricalDtype(ordered=False)) ref_modality_dependent = \ crosscount.loc[lambda df: df["overall_margin"] == df["overall_margin"].drop("overall_margin").max(), :] \ .index[0] new_levels = [ref_modality_dependent] + pd.CategoricalIndex(dependent_var_cat) \ .remove_categories(ref_modality_dependent) \ .categories.tolist() dependent_var_cat.cat.reorder_categories(new_levels, inplace=True) # MODEL # model = Logit(dependent_var_cat.cat.codes, independent_dummies) model = MNLogit(dependent_var_cat, independent_dummies) results = model.fit() params = results.params params.columns = dependent_var_cat.cat.categories[1:] params = params.rename_axis("dependent_var_modality") \ .drop("intercept", axis=0) \ .reset_index(drop=False) \ .melt(id_vars="dependent_var_modality", var_name="independent_var_modality") \ .assign(indicator="coeffs_LogR") pvalues = results.pvalues pvalues.columns = dependent_var_cat.cat.categories[1:] pvalues = pvalues.rename_axis("dependent_var_modality") \ .drop("intercept", axis=0) \ .reset_index(drop=False) \ .melt(id_vars="dependent_var_modality", var_name="independent_var_modality") \ .assign(indicator="pvalue_coeff_LogR") \ .round({"independent_var_modality": 2}) conf_int = results.conf_int() \ .reset_index(level=1, drop=False) \ .rename({"level_1": "independent_var_modality"}, axis=1) \ .loc[lambda df: df["independent_var_modality"] != "intercept", :] \ .rename_axis("dependent_var_modality") \ .reset_index(drop=False) \ .rename({"lower": "coeff_LogR_lb", "upper": "coeff_LogR_ub"}, axis=1) \ .melt(id_vars=["dependent_var_modality", "independent_var_modality"], value_vars=["coeff_LogR_lb", "coeff_LogR_ub"], var_name="indicator") LRT = pd.DataFrame.from_dict({ "dependent_var_modality": "overall_margin", "independent_var_modality": "overall_margin", "indicator": "pvalue_LRT_LogR", "value": results.llr_pvalue}, orient="index", ).transpose() chi2_pvalue = chi2_contingency(crosscount.drop("overall_margin", axis=0) \ .drop("overall_margin", axis=1))[1] chi2_crosscount = pd.DataFrame.from_dict({ "dependent_var_modality": "overall_margin", "independent_var_modality": "overall_margin", "indicator": "pvalue_chisquare_crosscount", "value": chi2_pvalue}, orient="index", ).transpose() multicategorical_multicategorical = pd.concat([params, pvalues, conf_int, LRT, chi2_crosscount], axis=0) \ .assign(ref_modality_dependent=ref_modality_dependent, ref_modality_independent=ref_modality_independent, independent_var_name=independent_var_name, dependent_var_name=dependent_var_name) return multicategorical_multicategorical[list_columns_names_export]
rf.score(X_test, y_test) sorted_import_index = rf.feature_importances_.argsort() """Compute Standard Errors""" from statsmodels.discrete.discrete_model import Logit, MNLogit top_topics = mean_diff.topic.values tmp = y.copy() for i in range(10): y_31 = pd.Series(np.where(tmp==top_topics[i],1,0)) sm_logit = Logit(endog=y_31, exog=X.reset_index(drop=True)) print(top_topics[i], '\n', sm_logit.fit().summary()) MN_logit = MNLogit(endog=y.astype(str), exog=X) MN_logit.fit(method='nm',maxiter=5000, maxfun=5000).summary() """Make Predictions at the means""" sample_data = pd.DataFrame(index=range(0,50), columns=["asian_proportion","latinx_proportion","log_income","log_price"]) sample_data[["asian_proportion","latinx_proportion","log_income","log_price"]] = df[["asian_proportion","latinx_proportion","log_income","log_price"]].mean().values.reshape(1,-1) b_min = df.black_proportion.min() b_max = df.black_proportion.max() sample_data["black_proportion"] = range(1,51) sample_data.black_proportion = ((b_max-b_min)/50*sample_data.black_proportion) sample_data = sample_data[["black_proportion","asian_proportion","latinx_proportion","log_income","log_price"]] predcited_values = LR.predict_proba(sample_data.values) [np.argmax(x) for x in predcited_values] df.to_csv('data/data514.csv')