def compproc(data, v): out = xyvars(data, v) xdata = out[0] ydata = out[1] abdata = absdiff(data, v, xdata, ydata) X = pd.get_dummies(data['round_age_x'], drop_first=True) xvar = v + '_x' X['Min_x'] = data['Min_x'] X['Min_y'] = data['Min_y'] X['traded'] = data['traded'] X = sm.add_constant(X) absstr = v + '_absdiff' mod = PanelOLS(data[absstr], X, entity_effects=True) res = mod.fit() print(res) params = res.params tradecoeff = params.loc['traded'] conf_int = res.conf_int() conf_int = conf_int.loc['traded'] lowconf = conf_int.iloc[0] upconf = conf_int.iloc[1] absstrm = v + '_absmean' absstrsd = v + '_abssd' absmean = data[absstrm].mean() abssd = data[absstrsd].mean() return ([tradecoeff, lowconf, upconf, absmean, abssd])
def kfoldfun(y, X, k): rng = np.random.RandomState(seed=12345) s = 100 seeds = np.arange(s) tot_error = 0 rng.shuffle(seeds) rsqtot = 0 for seed in seeds: cv = KFold(n_splits=k, shuffle=True, random_state=seed) for train_index, valid_index in cv.split(X, y): mod = PanelOLS(y.iloc[train_index], X.iloc[train_index], entity_effects=True) res = mod.fit(cov_type='clustered') pred = mod.predict(res.params, exog=X.iloc[valid_index]) rsq = 1 - (((y.iloc[valid_index].to_numpy() - pred.to_numpy().transpose())**2).sum()) / (( (y.iloc[valid_index].to_numpy() - y.iloc[valid_index].to_numpy().mean())**2).sum()) MSPE = np.abs((y.iloc[valid_index].to_numpy() - pred.to_numpy().transpose())).mean() tot_error = tot_error + MSPE rsqtot = rsqtot + rsq print("Mean Absolute Error:") print(tot_error / (s * k)) print("OOS R^2") print(rsqtot / (s * k))
def fixedEffects( self, y, x, id, year, entity_Effects=False, time_Effects=False, cov_Type="clustered", cluster_Entity=True, clean_data="greedy", ): if type(x) != str: utterance = ( "ERROR: Multiple independent regressor approach not yet implemented." ) return utterance s = self.map_column_to_sheet(y) # prepare data v = np.copy(x) v = np.append(v, y) df = s.cleanData(v, clean_data) # set up panel and return fit df = df.set_index([id, year]) mod = PanelOLS( df[y], df[x], entity_effects=entity_Effects, time_effects=time_Effects ) utterance = ( "Here are the results of a fixed effects regression of " + str(y) + " on " + str(x) ) utterance = ( utterance + ", using " + str(year) + " as the time dimension and " + str(id) + " as the id dimension.\n\n" ) utterance = utterance + str( mod.fit(cov_type=cov_Type, cluster_entity=cluster_Entity) ) return QueryResult( mod.fit(cov_type=cov_Type, cluster_entity=cluster_Entity), utterance )
def hausman_fe_re(panel_data, inef_formula, weights=None, cov="unadjusted", level=0.05): """ Executes a Hausman test, which H0: there is no correlation between unobserved effects and the independent variables It is not necessary to assign the function to an object! But remember to include an intercept in the formulas. :param panel_data : dataframe (which must be in a panel structure) :param inef_formula : patsy formula for the inefficient model under H0 (fixed effects) :param weights : N x 1 Series or vector containing weights to be used in estimation; defaults to None Use is recommended when analyzing survey data, passing on the weight available in the survey :param cov : str unadjusted: common standard errors robust: robust standard errors kernel: robust to heteroskedacity AND serial autocorrelation :param level : significance level for the test. Defaults to 5%. """ ## Random Effects if weights is None: random = RandomEffects.from_formula(formula=inef_formula, data=panel_data).fit(cov_type=cov) else: random = RandomEffects.from_formula(formula=inef_formula, data=panel_data, weights=weights).fit(cov_type=cov) ## Fixed Effects formula_fe = inef_formula + ' + EntityEffects' if weights is None: fixed = PanelOLS.from_formula(formula=formula_fe, data=panel_data, drop_absorbed=True).fit(cov_type=cov) else: fixed = PanelOLS.from_formula(formula=formula_fe, data=panel_data, drop_absorbed=True, weights=weights).fit(cov_type=cov) ## Computing the Hausman statistic # Difference between asymptotic variances var_assin = fixed.cov - random.cov # Difference between parameters d = fixed.params - random.params # Calculating H (statistic) H = d.dot(np.linalg.inv(var_assin)).dot(d) # Degrees of freedom freedom = random.params.size - 1 # Calculating p-value using chi2 survival function (sf, 1 - cumulative distribution function) p = stats.chi2(freedom).sf(H) if p < level: print(f"The value of H is {round(H, 6)} with {freedom} degrees of freedom in the chi-squared distribution.") print(f"The p-value of the test is {round(p, 6)} and, therefore, H0 is REJECTED and fixed effects is preferred") else: print(f"The value of H is {round(H, 6)} with {freedom} degrees of freedom in the chi-squared distribution.") print(f"The p-value of the test is {round(p, 6)} and H0 is NOT REJECTED and random effects is preferred.")
def POLS(data, y, xs, includeFixed=False, includeTime=False): # xs_str = ' + '.join(xs) # formula = f'{y} ~ {xs_str} + 1' # print(formula) # print(data['c10'].head()) # if includeFixed: # formula += '+ EntityEffects' # if includeTime: # formula += '+ TimeEffects' # mod = PanelOLS.from_formula(formula, data=data) # if includeFixed: # ori = mod.fit(cov_type='clustered', cluster_entity=True) # else: # ori = mod.fit() # print("Formula:"+formula) # print(ori.params) # print(ori.pvalues) # print(ori.rsquared_overall) # print('\n') # data = data.dropna() print(xs) exog = sm.add_constant(data[xs]) res = PanelOLS(data[y], exog, entity_effects=includeFixed, time_effects=includeTime).fit() return res
def panel_regression(self, X, y, entity_col, time_col, entity_effects=False, time_effects=False, other_effects=None, add_const=True, drop_absorbed=True): """ other_effects (array-like) – Category codes to use for any effects that are not entity or time effects. Each variable is treated as an effect return fitted res """ X = X.set_index([entity_col, time_col]) y.index = X.index if add_const: X = sm.add_constant(X) if other_effects is None: mod = PanelOLS(y, X, entity_effects=entity_effects, time_effects=time_effects)#, endog_names=['intercept'] + X.columns) else: mod = PanelOLS(y, X, entity_effects=entity_effects, time_effects=time_effects, other_effects=X[other_effects]) res = mod.fit() print(res.summary) return res
def run_regression( salesWithFlags, use_features=None, entity_effects=True, time_effects=True, cov_type="clustered", cluster_entity=True, ): """ Run a panel regression on the input sales data. Parameters ---------- salesWithFlags : pandas.DataFrame the sales data with any interaction flags already added use_features : list of str, optional if specified, only include these property characteristics in the regression entity_effects : bool, optional include neighborhood fixed effects time_effects : bool, optional include year fixed effects cov_type : str, optional the covariance type to use cluster_entity : bool, optional if using clustered errors, cluster at the neighborhood level """ from linearmodels import PanelOLS # get the modeling inputs X, Y = get_modeling_inputs(salesWithFlags, dropna=False, as_panel=True, use_features=use_features) # initialize the panel regression mod = PanelOLS(Y, X, entity_effects=entity_effects, time_effects=time_effects) # return the regression result return mod.fit(cov_type=cov_type, cluster_entity=cluster_entity)
def Reg_Painel_Efeitos_Fixos(x, y, constante="S", cov='normal'): ''' Função que calcula uma regressão de efeitos fixos, sendo, por default, computada com um intercepto e com erros padrões não robustos. **IMPORTANTE: para o painel estar arrumado, os dados devem estar multi-indexados por indíviduo e por tempo, nesta ordem. Caso contrário, transformar o dataframe usando a função 'Arrumar Painel' x: lista ou array com os valores das variáveis independentes; y: lista ou array com os valores da variável dependente; constante: "S" para regressão com intercepto e qualquer outro valor para sem intercepto. Caso em branco, a regressão é computada com intercepto; cov: "normal" para regressão com erros-padrão tradicionais (caso padrão); "robust" para erros-padrões robustos. "cluster" ou "clustered" para erros-padrões clusterizados ''' global df, Resultado # formando o vetor de variáveis independentes if constante == "S": X = sm.add_constant(x) else: X = x #Criando o Modelo levando em conta a opção dos erros padrão Modelo = PanelOLS(y, X, entity_effects=True, drop_absorbed=True) if cov == "robust": Resultado = Modelo.fit(cov_type='robust') elif cov == 'kernel': ## correlação robusta à heteroscedasticidade e autocorrelação serial Resultado = Modelo.fit(cov_type='kernel') elif cov == 'clustered' or cov == 'cluster': Resultado = Modelo.fit(cov_type='clustered', cluster_entity=True) else: Resultado = Modelo.fit() print(Resultado)
def fixed_effects(panel_data, formula, weights=None, time_effects=False, cov="unadjusted"): """ Fits a standard Fixed Effects model with the corresponding covariance matrix. It can be estimated WITH and WITHOUT a constant. It is preferred when the unobserved effects are correlated with the error term and, therefore, CAN'T estimate constant terms. Remember to include an intercept in the formula ('y ~ 1 + x1 + ...') and to assign it to an object! :param panel_data : dataframe (which must be in a panel structure) :param formula : patsy/R formula (without EntityEffects, will be added inside the function) :param weights : N x 1 Series or vector containing weights to be used in estimation; defaults to None Use is recommended when analyzing survey data, passing on the weight available in the survey :param time_effects : bool, defaults to False Whether to include time effects alongside entity effects (and estimate a two-way fixed effects) :param cov : str unadjusted: common standard errors robust: robust standard errors kernel: robust to heteroskedacity AND serial autocorrelation clustered: clustered standard errors by the entity column :return : linearmodels model instance """ ## Creating model instance # Defining which effects to control for formula += ' + EntityEffects + TimeEffects' if time_effects else ' + EntityEffects' ## Creating model instance if weights is None: mod = PanelOLS.from_formula(formula=formula, data=panel_data, drop_absorbed=True) else: mod = PanelOLS.from_formula(formula=formula, data=panel_data, drop_absorbed=True, weights=weights) ## Fitting with desired covariance matrix mod = mod.fit(cov_type='clustered', cluster_entity=True) if cov == 'clustered' else mod.fit(cov_type=cov) print(mod.summary) return mod
def one_step_panel_fit(data): """ Panel regression is exactly the same as pooled regression!!! All coefficients estimation are the same """ fit = PanelOLS( data['ret'], data[[ 'const', 'market_cap', 'pe', 'pe_lyr', 'pb', 'ps', 'pcf', 'turnover' ]]).fit() logger.info("Panel Regression") logger.info(fit) resid = fit.resids logger.info("Residual auto correlation") logger.info( format_for_print( pd.DataFrame( [resid.autocorr(1), resid.autocorr(5), resid.autocorr(20)]))) return resid
def run_regression(df): df = df.set_index(['county_id', 'year']) model = PanelOLS.from_formula('chips_sold ~ 1 + post_tv + EntityEffects + TimeEffects', data = df) fit = model.fit() return(fit)
def estimate_profiles(graphs=False): ''' Function to estimate deterministic lifecycle profiles of hourly earnings. Follows methodology of Fullerton and Rogers (1993). Args: graphs (bool): whether to create graphs of profiles Returns: reg_results (Pandas DataFrame): regression model coefficients for lifetime earnings profiles ''' # Read in dataframe of PSID data df = ogusa.utils.safe_read_pickle( os.path.join(cur_path, 'data', 'PSID', 'psid_lifetime_income.pkl')) model_results = { 'Names': [ 'Constant', '', 'Head Age', '', 'Head Age^2', '', 'Head Age^3', '', 'R-Squared', 'Observations' ] } cats_pct = ['0-25', '26-50', '51-70', '71-80', '81-90', '91-99', '100'] long_model_results = { 'Lifetime Income Group': [], 'Constant': [], 'Age': [], 'Age^2': [], 'Age^3': [], 'Observations': [] } for i, group in enumerate(cats_pct): data = df[df[group] == 1].copy() data['ones'] = np.ones(len(data.index)) mod = PanelOLS(data.ln_earn_rate, data[['ones', 'age', 'age2', 'age3']]) res = mod.fit(cov_type='clustered', cluster_entity=True) # print('Summary for lifetime income group ', group) # print(res.summary) # Save model results to dictionary model_results[group] = [ res.params['ones'], res.std_errors['ones'], res.params['age'], res.std_errors['age'], res.params['age2'], res.std_errors['age2'], res.params['age3'], res.std_errors['age3'], res.rsquared, res.nobs ] long_model_results['Lifetime Income Group'].extend([cats_pct[i], '']) long_model_results['Constant'].extend( [res.params['ones'], res.std_errors['ones']]) long_model_results['Age'].extend( [res.params['age'], res.std_errors['age']]) long_model_results['Age^2'].extend( [res.params['age2'], res.std_errors['age2']]) long_model_results['Age^3'].extend( [res.params['age3'], res.std_errors['age3']]) long_model_results['Observations'].extend([res.nobs, '']) reg_results = pd.DataFrame.from_dict(model_results) reg_results.to_csv( os.path.join(output_dir, 'DeterministicProfileRegResults.csv')) long_reg_results = pd.DataFrame.from_dict(model_results) long_reg_results.to_csv( os.path.join(output_dir, 'DeterministicProfileRegResults_long.csv')) if graphs: # Plot lifecycles of hourly earnings from processes estimated above age_vec = np.arange(20, 81, step=1) for i, group in enumerate(cats_pct): earn_profile = (model_results[group][0] + model_results[group][2] * age_vec + model_results[group][4] * age_vec**2 + model_results[group][6] * age_vec**3) plt.plot(age_vec, earn_profile, label=group) plt.title( 'Estimated Lifecycle Earnings Profiles by Lifetime Income Group') plt.legend() plt.savefig(os.path.join(output_dir, 'lifecycle_earnings_profiles.png')) # Plot of lifecycles of hourly earnings from processes from data pd.pivot_table(df, values='ln_earn_rate', index='age', columns='li_group', aggfunc='mean').plot(legend=True) plt.title( 'Empirical Lifecycle Earnings Profiles by Lifetime Income Group') plt.savefig( os.path.join(output_dir, 'lifecycle_earnings_profiles_data.png')) # Plot of lifecycle profiles of hours by lifetime income group # create variable from fraction of time endowment work df['labor_supply'] = (df['earnhours_hh'] / (24 * 5 * (df['married'] + 1) * 50)) pd.pivot_table(df, values='labor_supply', index='age', columns='li_group', aggfunc='mean').plot(legend=True) plt.title('Lifecycle Profiles of Hours by Lifetime Income Group') plt.savefig(os.path.join(output_dir, 'lifecycle_laborsupply.png')) return reg_results
gaps.loc[gaps.index, "indcom4"] = 0 gaps.loc[gaps.t == 4, "indcom4"] = 1 gaps.loc[gaps.index, "indcom6"] = 0 gaps.loc[gaps.t == 6, "indcom6"] = 1 gaps = gaps.loc[~gaps.State.isin([ "Alaska", "Delaware", "Montana", "North Dakota", "South Dakota", "Vermont", "Wyoming" ])] gaps.set_index(["State", "Year"], inplace=True) gaps["gap"] = gaps["gap"].abs() model = PanelOLS.from_formula( 'gap ~ 1 + indcom_4 + indcom_2 + indcom + indcom2 + indcom4 + indcom6', data=gaps) print(model.fit(cov_type="robust")) ########### ###STATE### ########### starts = pd.read_excel( "/home/matt/GitRepos/ElectionData/data/Independent_Commission_Start.xlsx", "Sheet1", skip_footer=2) starts["time"] = 1 gaps = get_efficiency_gap("federal")[['State', 'Year', 'gap']]
#%% import numpy as np from statsmodels.datasets import grunfeld data = grunfeld.load_pandas().data data.year = data.year.astype(np.int64) # MultiIndex, entity - time data = data.set_index(['firm', 'year']) from linearmodels import PanelOLS mod = PanelOLS(data.invest, data[['value', 'capital']], entity_effects=True) res = mod.fit(cov_type='clustered', cluster_entity=True) #%% from linearmodels import PanelOLS mod = PanelOLS.from_formula('invest ~ value + capital + EntityEffects', data) res = mod.fit(cov_type='clustered', cluster_entity=True) #%%
# Create indicator variables for Difference in Difference conflict["PostConflict"] = conflict['Year'].apply(lambda x: 1 if x >= 2014 else 0) conflict['Treated'] = conflict['intensity'].apply(lambda x: 1 if x > 1 else 0) # Conduct base Difference in Difference BaseModel = smf.ols("Pop_percent_change ~ Treated * PostConflict ", data=conflict).fit() print(BaseModel.summary()) # Difference in Difference with Confounding Factors CFModel = smf.ols( "Pop_percent_change ~ Treated * PostConflict + Hospitals + Population_Percent_Child + Population_Percent_Female + Poverty_Rate + Airport", data=conflict).fit() print(CFModel.summary()) # Difference in Difference by County CountyModel = smf.ols( "Pop_percent_change ~ C(County) + Treated * PostConflict", data=conflict).fit() print(CountyModel.summary()) # Panel OLS conflict = conflict.set_index(['County', 'Year']) PanelModel = PanelOLS.from_formula( 'Pop_percent_change ~ Treated * PostConflict + EntityEffects', data=conflict, drop_absorbed=True) PanelModel.fit(cov_type='clustered', cluster_entity=True)
], 'Single Males': [], 'Single Females': [], 'Married, Male Head': [], 'Married, Female Head': [] } for i, data in enumerate(list_of_dfs): # Note that including entity and time effects leads to a collinearity # I think this is because there are some years at begin and end of # sample with just one person # mod = PanelOLS(data.ln_wage_rate, # data[['age', 'age2', 'age3']], # weights=data.fam_smpl_wgt_core, # entity_effects=True, time_effects=True) mod = PanelOLS(data.ln_wage_rate, data[['age', 'age2', 'age3']], entity_effects=True) res = mod.fit(cov_type='clustered', cluster_entity=True) print('Summary for ', list_of_statuses[i]) print(res.summary) # Save model results to dictionary first_stage_model_results[list_of_statuses[i]] = [ res.params['age'], res.std_errors['age'], res.params['age2'], res.std_errors['age2'], res.params['age3'], res.std_errors['age3'], res.rsquared, res.nobs, res.entity_info['total'] ] fit_values = res.predict(fitted=True, effects=True, missing=True) fit_values['predictions'] = (fit_values['fitted_values'] + fit_values['estimated_effects']) list_of_dfs_with_fitted_vals.append( data.join(fit_values, how='left', on=['hh_id', 'year']))
if cluster_type in ('random', 'other-random', 'entity-nested', 'random-nested'): clusters = y.copy() if cluster_type == 'random': clusters.dataframe.iloc[:, :] = random_effects elif cluster_type == 'other-random': clusters.dataframe.iloc[:, :] = other_random elif cluster_type == 'entity_nested': eid = y.entity_ids clusters.dataframe.iloc[:, :] = eid // 3 elif cluster_type == 'random-nested': clusters.dataframe.iloc[:, :] = random_effects // 2 fo['clusters'] = clusters mod = PanelOLS(data.y, data.x, **mo) res = mod.fit(**fo) res2 = mod.fit(auto_df=False, count_effects=False, **fo) res3 = mod.fit(auto_df=False, count_effects=True, **fo) res4 = mod.fit(cov_type='unadjusted') res5 = mod.fit(cov_type='unadjusted', auto_df=False, count_effects=False) res6 = mod.fit(cov_type='unadjusted', auto_df=False, count_effects=True) vals[b] = np.column_stack([ res.params, res.std_errors, res2.std_errors, res3.std_errors, res4.std_errors, res5.std_errors, res6.std_errors ])
dfProvince['Ml2_cat1'] = dfProvince['mosquito_lag2'] < q1 dfProvince['Ml2_cat2'] = (dfProvince['mosquito_lag2'] > q1) & (dfProvince['mosquito_lag1'] < q2) dfProvince['Ml2_cat3'] = (dfProvince['mosquito_lag2'] > q2) & (dfProvince['mosquito_lag1'] < q3) dfProvince['Ml2_cat4'] = dfProvince['mosquito_lag2'] > q3 #take log of dengue and add its lag in data frame dfProvince['log_dengue'] = np.log(dfProvince['Dengue'] + 1) dfProvince['lag_log_dengue'] = dfProvince['log_dengue'].shift(1) #-------------------- Model --------------------------------------------- #first specification X_spec1 = sm.add_constant( dfProvince.loc[:, ['lag_log_dengue', 'M_cat2', 'M_cat3', 'M_cat4']]) mod1 = PanelOLS(dfProvince['log_dengue'], X_spec1, entity_effects=True) res1 = mod1.fit(cov_type='clustered') print(res1) #second specification with kfold X_spec2 = sm.add_constant(dfProvince.loc[:, [ 'M_cat2', 'M_cat3', 'M_cat4', 'Ml_cat2', 'Ml_cat3', 'Ml_cat4', 'Ml2_cat2', 'Ml2_cat3', 'Ml2_cat4' ]]) mod2 = PanelOLS(dfProvince['log_dengue'], X_spec2, entity_effects=True) res2 = mod2.fit(cov_type='clustered') print(res2) #third specification with kfold X_spec3 = sm.add_constant(dfProvince.loc[:, [
from pathlib import Path import numpy as np import pandas as pd import statsmodels.api as sm from studies.age_structure.commons import * from linearmodels import PanelOLS # dcons = average daily consumption per household # rc = percent change in daily consumption per household relative to 2019m6 df = pd.read_stata("data/datareg.dta") index = ["districtnum", "month_code"] rc = df[index + ["rc"]].set_index(index) exog_cols = ["I_cat", "D_cat", "I_cat_national", "D_cat_national"] for col in exog_cols: df[col] = pd.Categorical(df[col]) exog = df[index + exog_cols].set_index(index) PanelOLS(rc, exog, entity_effects = True)
def Panel_output(endo, exog): X = sm.add_constant(df.loc[:, exog]) mod = PanelOLS(df['log_dengue'], X, entity_effects=True) res = mod.fit(cov_type='clustered') print(res) return (res.loglik, exog, res)
col='country', hue='country', col_wrap=4, palette="deep") SRPC_other = SRPC_other.map(plt.plot, 'unemployment', 'inflation').set_titles("{col_name}") ######## e. Panel data regression analysis ####### #Panel data regression for full sample merge_eu = merge_eu.reset_index() year_full = pd.Categorical(merge_eu.year) merge_eu = merge_eu.set_index(['country', 'year']) merge_eu['year'] = year_full regression1 = PanelOLS(merge_eu.inflation, merge_eu.unemployment, entity_effects=True) res1 = regression1.fit(cov_type='clustered', cluster_entity=True) print(res1) # Panel data regression for data after QE after_QE = after_QE.reset_index() year_QE = pd.Categorical(after_QE.year) after_QE = after_QE.set_index(['country', 'year']) after_QE['year'] = year_QE regression2 = PanelOLS(after_QE.inflation, after_QE.unemployment, entity_effects=True) res2 = regression2.fit(cov_type='clustered', cluster_entity=True) print(res2)
BetweenModel = BetweenOLS.from_formula('fcs ~ rev_percap + month_Decembre', data=data, weights=w) BetweenModel.fit(cov_type='robust', reweight=True) # RANDOM EFFECTS RandomEffectsModel = RandomEffects.from_formula( 'fcs ~ rev_percap + year + month_Decembre', data=data, weights=w) REModFit = RandomEffectsModel.fit(cov_type='robust') REModFit REModFit.variance_decomposition REModFit.theta # BASIC PANEL PanelModel = PanelOLS.from_formula( 'fcs ~ 1 + rev_percap + month_Decembre + EntityEffects', data=data, weights=w) PanelModel.fit(cov_type='robust') # INTERPRETATION : TO BE FULLY CHECKED # une augmentation de 1000 du revenu par rapport à sa moyenne sur a période # augmente de X le score fcs par rapport à sa moyenne sur a période # # ESTIMATION EXCLUDING DECEMBER # datajun = data[data['month'].isin(['Juin'])].reset_index(drop=False) datajun = datajun.drop(columns={'time'}) time_df = datajun[['year', 'month']].drop_duplicates() time_df = time_df.sort_values('month', ascending=False).sort_values('year')
## X x_list = ['ls_num', 'lti', 'ln_loanamout', 'ln_appincome', 'subprime', 'secured', \ 'cb', 'ln_ta', 'ln_emp', 'num_branch', 'ln_pop', 'density', 'hhi', 'ln_mfi',\ 'mean_distance'] x = df[x_list] ''' x_msat_list = x_list + ['dum_msat_{}'.format(i) for i in range(dum_msat.shape[1])] x_msat = sm.add_constant(df[x_msat_list]) ''' #------------------------------------------------------------ # Run regression #------------------------------------------------------------ # Run no dum res_nd = PanelOLS(y, x).fit(cov_type='clustered', cluster_entity=True) ## Save output to txt text_file = open("Results/Results_baseline_nodummy.txt", "w") text_file.write(res_nd.summary.as_text()) text_file.close() # Run dum_t res_t = PanelOLS(y, x, entity_effects=True, time_effects=True).fit(cov_type='clustered', cluster_entity=True) ## Save output to txt text_file = open("Results/Results_baseline_t.txt", "w") text_file.write(res_t.summary.as_text()) text_file.close()
def panel_data(train, years_ahead=1): """ It uses a random forest trained on the observed values of a data matrix (selected series codes except those in submit_rows_index) to predict the missing values. after that, use panel data model for prediction Returns: y_pred: prediction values of target """ train_melt = pd.melt(train.iloc[:, 0:38], id_vars=['Country Name', 'Series Code'], value_vars=train.columns[0:36], var_name='year', value_name='value') train_melt['year'] = train_melt['year'].str[:4].astype(int) panel = train_melt.groupby(['Country Name', 'year', 'Series Code'])['value'].mean().unstack() # only use code with at least one observed value across 36 years in each country for the imputation data matrix left_feature = panel.iloc[:, 9:].isna().groupby('Country Name').sum().max( axis=0) <= 18 pred = panel.iloc[:, 9:].iloc[:, left_feature.values] # construct matrix of features across countries df = [] ct_list = list(set(pred.index.get_level_values(0))) ct_list = sorted(ct_list) for i in ct_list: df.append(pred.loc[i]) predictors = pd.concat(df, axis=1) # random forest imputation imputer = MissForest() predictors_imputed = imputer.fit_transform(predictors) panel.reset_index(inplace=True) panel.columns = ['Country Name', 'year'] + [ 'y' + str(i) for i in range(1, 10) ] + ['x' + str(i) for i in range(1, 1297)] nfeature = int(predictors.shape[1] / 214) split = list(range(nfeature, predictors_imputed.shape[1], nfeature)) _ = np.split(predictors_imputed, split, 1) predictors_new = pd.DataFrame(np.vstack(_)) predictors_new['year'] = panel.year predictors_new['Country Name'] = panel['Country Name'] predictors_new.columns = [ 'x' + str(i) for i in range(1, pred.shape[1] + 1) ] + ['year', 'Country Name'] # combine the updated feature matrix and responses feature = predictors_new.isna().sum() <= 0 # change to 1 panel_left = predictors_new.iloc[:, feature.values] panel_comb = pd.merge(panel.iloc[:, 0:11], panel_left.shift(years_ahead)) # Split prediction and target panel_train = panel_comb.loc[panel_comb.year < 2007] panel_train = panel_train.set_index(['Country Name', 'year']) panel_test = panel_comb.loc[panel_comb.year == 2007] panel_test = panel_test.set_index(['Country Name', 'year']) # panel data model with warnings.catch_warnings(): warnings.filterwarnings("ignore") Ypred = pd.DataFrame() for i in range(1, 10): formula = 'y' + str(i) + '~1+' + '+'.join( panel_train.columns[11:].values) + '+EntityEffects' mod = PanelOLS.from_formula(formula, panel_train) res = mod.fit(cov_type='clustered', cluster_entity=True) Ypred['y' + str(i)] = res.predict(data=panel_test).predictions # Eval Yval = panel_test.iloc[:, :9] rmse = np.sqrt(np.nanmean(np.power(Ypred - Yval, 2))) print(rmse) return Ypred
import os from statsmodels.iolib.summary2 import summary_col import matplotlib.pyplot as plt import seaborn as sns import statsmodels.api as sm from linearmodels import PanelOLS from linearmodels import RandomEffects if __name__ == "__main__": REG_DATA = sys.argv[1] RES3_PATH = sys.argv[2] metadata = pd.read_csv(REG_DATA) metadata = metadata.sort_values(by=['Code', 'Year']) metadata = metadata.set_index(['Code', 'Year']) metadata['Income_t0_log'] = np.log10(metadata['Income_t0']) base = os.path.basename(RES3_PATH) incomegroup = base.split(".")[0].split("_")[-1] metadata = metadata[metadata.IncomeGroup == incomegroup] metadata = metadata.dropna() num_period = len(metadata['period'].unique()) metadata = metadata[metadata['size'] == num_period] exog_vars = ['ECI', 'Income_t0_log', 'diversity'] exog = sm.add_constant(metadata[exog_vars]) mod = PanelOLS(metadata.growth, exog, entity_effects=True) with open(RES3_PATH, 'w') as f: f.write(mod.fit().summary.as_text())
from matplotlib.backends.backend_pdf import PdfPages from linearmodels import PanelOLS #import data data = pd.DataFrame.from_csv("fraserDataWithRGDPPC.csv", index_col=[0, 1], parse_dates=True) # create list of each index set from multi index years = list(sorted(set(data.index.get_level_values('Year')))) country = list(sorted(set(data.index.get_level_values('ISO_Code')))) #choose variables that will be plotted for each year in scatter plot_vars = [ "Sound Money", "Government Consumption", "RGDP Per Capita", "Quartile" ] # Normalize income so that 1 represents the maximum value of RGDP Per Capita # This will allow dot to be easily adjusted data["RGDP Per Capita"] = data["RGDP Per Capita"] / max( data["RGDP Per Capita"]) * 1000 # Panel OLS reg_data = data[[ "RGDP Per Capita", "Sound Money", "Government Consumption", "SUMMARY INDEX" ]].dropna() x = reg_data[["Sound Money", "Government Consumption", "SUMMARY INDEX"]] y = reg_data[["RGDP Per Capita"]] mod = PanelOLS(y, x, entity_effects=True, time_effects=False) res = mod.fit(cov_type='clustered', cluster_entity=True) print(res.summary)
merge_tot = merge_tot.set_index(["省份", "日期"]) merge_norm = (merge_tot - merge_tot.mean()) / (merge_tot.max() - merge_tot.min()) print(merge_norm) from linearmodels import PanelOLS y = merge_norm[["新增确诊"]] x = merge_norm[[ "_1traffic", "_2traffic", "_3traffic", "traffic", "traffic3_", "traffic2_", "traffic1_", ]] #change here reg = PanelOLS(y, x, entity_effects=True, time_effects=True) res = reg.fit(cov_type='clustered', cluster_entity=True) print(res) parameters = [0.0433, 0.0231, 0.0075, 0.0176, 0.0053, 0.0034, 0.0086] xline = [-3, -2, -1, 0, 1, 2, 3] lower = [-0.0151, 0.0040, 0.0075, 0.0007, -0.0108, -0.0162, -0.017] upper = [0.1017, 0.0422, 0.0075, 0.0346, 0.0214, 0.0229, 0.0342] for i in range(len(parameters)): parameters[i] /= 0.0075 lower[i] /= 0.0075 upper[i] /= 0.0075 import matplotlib.pyplot as plt plt.plot(xline, parameters, marker="*", color="black") for i in range(len(xline)): plt.vlines(x=xline[i], ymin=lower[i], ymax=upper[i], label="*") plt.vlines(x=0, ymin=-2, ymax=6.0, linestyles="dashed")
# Set vars ## Y y = 'log_min_distance' ## X x_list = ['ls_num', 'lti', 'ln_loanamout', 'ln_appincome', 'subprime', 'secured', \ 'cb', 'ln_ta', 'ln_emp', 'num_branch', 'ln_pop', 'density', 'hhi', 'ln_mfi',\ 'mean_distance'] x = ' + '.join(x_list) #------------------------------------------------------------ # Run regressions #------------------------------------------------------------ # Run Bank + msat dummies res_msat = PanelOLS.from_formula('{} ~ {}'.format(y,x), data = df_msat).fit(cov_type = 'clustered', cluster_entity = True) ## Save output to txt text_file = open("Results/Results_baseline_msat.txt", "w") text_file.write(res_msat.summary.as_text()) text_file.close() # Run Bankmsa + t dummies res_msabank = PanelOLS.from_formula('{} ~ {}'.format(y,x), data = df_msabank).fit(cov_type = 'clustered', cluster_entity = True) ## Save output to txt text_file = open("Results/Results_baseline_msabank.txt", "w") text_file.write(res_msabank.summary.as_text()) text_file.close() # Run Bank + t dummies
def fit(self, X, y, entity_effects=True, weekday_effects=True, cov_type='clustered'): """ Parameters ---------- X : Pandas DataFrame Panel DataFrame of entities observed at multiple points in time. y : str Column to be used as regression target. entity_effects : bool, default True If True, include entity fixed effects into the model. If False, the estimation procedure is equivalent to pooled OLS. weekday_effects : bool, default True If True, include a dummy for each day of the week. Due to the large variance in activity features between weekdays, for certain situations this is highly recommended. cov_type : str, default 'clustered' Covariance matrix structure. Must be one of 'clustered', 'robust'. Note if entity_effects is set to True, robust standard errors are no longer robust. Returns ------- self.regression_results_ : linearmodels.panel.results.PanelEffectsResults Summary of estimation results. """ self._depvar_label = ' '.join([w.capitalize() for w in y.split('_')]) idx_cols = [self.entity_col, self.time_col] relative_idx = ((X[self.time_col] - X[self.event_col]) / dt.timedelta(days=1)).astype(int) dummies = onehot_integer_series(relative_idx) # Add in dummy variables for observation distance to event X = pd.concat([X[[self.entity_col, self.time_col, y]], dummies], axis=1) # Set our estimation target indvars = list(dummies.columns) if weekday_effects: X['day_of_week'] = X[self.time_col].dt.strftime('%A') indvars = indvars + ['day_of_week'] X.set_index(idx_cols, inplace=True) X.sort_index(inplace=True) depvar = X[y] model = PanelOLS(dependent=depvar, exog=X[indvars], entity_effects=entity_effects) self.regression_results_ = model.fit(cov_type='clustered') # Extract point estimates coefs = self.regression_results_.params.reset_index() coefs = coefs[coefs['index'].str.contains('relative_idx')] coefs['index'] = coefs['index'].apply(self.parse_dummies) coefs.sort_values('index', inplace=True) self._idx_coefs = coefs.rename(columns={ 'index': 'relative_idx' }).set_index('relative_idx') # Extract integer index, we can just use the coef index since cis are the same indexing self._event_relative_idx = coefs['index'].values # Extract confidence intervals cis = self.regression_results_.conf_int().reset_index() cis = cis[cis['index'].str.contains('relative_idx')] cis['index'] = cis['index'].apply(self.parse_dummies) cis.sort_values('index', inplace=True) self._idx_cis = cis.rename(columns={ 'index': 'relative_idx' }).set_index('relative_idx') return self.regression_results_
def estimate_profiles(graphs=False): """ Function to estimate deterministic lifecycle profiles of hourly earnings. Follows methodology of Fullerton and Rogers (1993). Args: graphs (bool): whether to create graphs of profiles Returns: reg_results (Pandas DataFrame): regression model coefficients for lifetime earnings profiles """ # Read in dataframe of PSID data df = ogusa.utils.safe_read_pickle( os.path.join(cur_path, "data", "PSID", "psid_lifetime_income.pkl")) model_results = { "Names": [ "Constant", "", "Head Age", "", "Head Age^2", "", "Head Age^3", "", "R-Squared", "Observations", ] } cats_pct = ["0-25", "26-50", "51-70", "71-80", "81-90", "91-99", "100"] long_model_results = { "Lifetime Income Group": [], "Constant": [], "Age": [], "Age^2": [], "Age^3": [], "Observations": [], } for i, group in enumerate(cats_pct): data = df[df[group] == 1].copy() data["ones"] = np.ones(len(data.index)) mod = PanelOLS(data.ln_earn_rate, data[["ones", "age", "age2", "age3"]]) res = mod.fit(cov_type="clustered", cluster_entity=True) # print('Summary for lifetime income group ', group) # print(res.summary) # Save model results to dictionary model_results[group] = [ res.params["ones"], res.std_errors["ones"], res.params["age"], res.std_errors["age"], res.params["age2"], res.std_errors["age2"], res.params["age3"], res.std_errors["age3"], res.rsquared, res.nobs, ] long_model_results["Lifetime Income Group"].extend([cats_pct[i], ""]) long_model_results["Constant"].extend( [res.params["ones"], res.std_errors["ones"]]) long_model_results["Age"].extend( [res.params["age"], res.std_errors["age"]]) long_model_results["Age^2"].extend( [res.params["age2"], res.std_errors["age2"]]) long_model_results["Age^3"].extend( [res.params["age3"], res.std_errors["age3"]]) long_model_results["Observations"].extend([res.nobs, ""]) reg_results = pd.DataFrame.from_dict(model_results) reg_results.to_csv( os.path.join(output_dir, "DeterministicProfileRegResults.csv")) long_reg_results = pd.DataFrame.from_dict(model_results) long_reg_results.to_csv( os.path.join(output_dir, "DeterministicProfileRegResults_long.csv")) if graphs: # Plot lifecycles of hourly earnings from processes estimated above age_vec = np.arange(20, 81, step=1) for i, group in enumerate(cats_pct): earn_profile = (model_results[group][0] + model_results[group][2] * age_vec + model_results[group][4] * age_vec**2 + model_results[group][6] * age_vec**3) plt.plot(age_vec, earn_profile, label=group) plt.title( "Estimated Lifecycle Earnings Profiles by Lifetime Income Group") plt.legend() plt.savefig(os.path.join(output_dir, "lifecycle_earnings_profiles.png")) # Plot of lifecycles of hourly earnings from processes from data pd.pivot_table( df, values="ln_earn_rate", index="age", columns="li_group", aggfunc="mean", ).plot(legend=True) plt.title( "Empirical Lifecycle Earnings Profiles by Lifetime Income Group") plt.savefig( os.path.join(output_dir, "lifecycle_earnings_profiles_data.png")) # Plot of lifecycle profiles of hours by lifetime income group # create variable from fraction of time endowment work df["labor_supply"] = df["earnhours_hh"] / (24 * 5 * (df["married"] + 1) * 50) pd.pivot_table( df, values="labor_supply", index="age", columns="li_group", aggfunc="mean", ).plot(legend=True) plt.title("Lifecycle Profiles of Hours by Lifetime Income Group") plt.savefig(os.path.join(output_dir, "lifecycle_laborsupply.png")) return reg_results