def test_read_dta18(self):
parsed_118 = self.read_dta(self.dta22_118)
parsed_118["Bytes"] = parsed_118["Bytes"].astype('O')
expected = DataFrame.from_records(
[['Cat', 'Bogota', u'Bogotá', 1, 1.0, u'option b Ünicode', 1.0],
['Dog', 'Boston', u'Uzunköprü', np.nan, np.nan, np.nan, np.nan],
['Plane', 'Rome', u'Tromsø', 0, 0.0, 'option a', 0.0],
['Potato', 'Tokyo', u'Elâzığ', -4, 4.0, 4, 4],
['', '', '', 0, 0.3332999, 'option a', 1/3.]
],
columns=['Things', 'Cities', 'Unicode_Cities_Strl', 'Ints', 'Floats', 'Bytes', 'Longs'])
expected["Floats"] = expected["Floats"].astype(np.float32)
for col in parsed_118.columns:
tm.assert_almost_equal(parsed_118[col], expected[col])
rdr = StataReader(self.dta22_118)
vl = rdr.variable_labels()
vl_expected = {u'Unicode_Cities_Strl': u'Here are some strls with Ünicode chars',
u'Longs': u'long data',
u'Things': u'Here are some things',
u'Bytes': u'byte data',
u'Ints': u'int data',
u'Cities': u'Here are some cities',
u'Floats': u'float data'}
tm.assert_dict_equal(vl, vl_expected)
self.assertEqual(rdr.data_label, u'This is a Ünicode data label')
def read_stata(self, *args, **kwargs): reader = StataReader(*args, **kwargs) self.df = reader.data() self.variable_labels = reader.variable_labels() self._initialize_variable_labels() self.value_labels = reader.value_labels() # self.data_label = reader.data_label() return self.df
def test_data_method(self):
# Minimal testing of legacy data method
reader_114 = StataReader(self.dta1_114)
with warnings.catch_warnings(record=True) as w:
parsed_114_data = reader_114.data()
reader_114 = StataReader(self.dta1_114)
parsed_114_read = reader_114.read()
tm.assert_frame_equal(parsed_114_data, parsed_114_read)
def test_read_dta1(self):
reader = StataReader(self.dta1)
parsed = reader.data()
# Pandas uses np.nan as missing value. Thus, all columns will be of type float, regardless of their name.
expected = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)],
columns=['float_miss', 'double_miss', 'byte_miss', 'int_miss', 'long_miss'])
for i, col in enumerate(parsed.columns):
np.testing.assert_almost_equal(
parsed[col],
expected[expected.columns[i]]
)
def test_read_dta1(self):
reader = StataReader(self.dta1)
parsed = reader.data()
# Pandas uses np.nan as missing value.
# Thus, all columns will be of type float, regardless of their name.
expected = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)],
columns=['float_miss', 'double_miss', 'byte_miss',
'int_miss', 'long_miss'])
# this is an oddity as really the nan should be float64, but
# the casting doesn't fail so need to match stata here
expected['float_miss'] = expected['float_miss'].astype(np.float32)
tm.assert_frame_equal(parsed, expected)
def _retrieve_data(dtafile):
'''retrieve data dictionary from STATA .dta file'''
datafile = os.path.basename(dtafile).split('.')
if len(datafile) != 2:
raise ValueError('dtafile must look like "file.dta"')
if datafile[1] != 'dta':
raise ValueError('dtafile must have ".dta" extension')
base = datafile[0]
hdf = os.path.join('.data_cache', '{}.h5'.format(base))
lPickle = os.path.join('.data_cache', '{}_labels.pickle'.format(base))
vPickle = os.path.join('.data_cache', '{}_vlabels.pickle'.format(base))
dTime = os.path.join('.data_cache', '{}_dtime.pickle'.format(base))
if all([os.path.isfile(d) for d in [hdf, lPickle, vPickle, dTime]]):
if os.path.getmtime(dtafile) == cPickle.load(open(dTime, 'rb')):
from pandas import read_hdf
data = read_hdf(hdf, 'data')
labels = cPickle.load(open(lPickle, 'rb'))
vlabels = cPickle.load(open(vPickle, 'rb'))
elif not os.path.isdir('.data_cache'):
os.makedirs('.data_cache')
try:
data
except:
from pandas.io.stata import StataReader
from pandas import HDFStore
print "Data is changed or no cached data found"
print "Creating data objects from {}".format(dtafile)
reader = StataReader(dtafile)
data = reader.data(convert_dates=False,convert_categoricals=False)
labels = reader.variable_labels()
vlabels = reader.value_labels()
store = HDFStore(hdf)
store['data'] = data
cPickle.dump(labels, open(lPickle, 'wb'))
cPickle.dump(vlabels, open(vPickle, 'wb'))
cPickle.dump(os.path.getmtime(dtafile), open(dTime, 'wb'))
store.close()
return {'data':data, 'labels':labels, 'vlabels':vlabels}
def test_read_dta1(self):
reader_114 = StataReader(self.dta1_114)
parsed_114 = reader_114.data()
reader_117 = StataReader(self.dta1_117)
parsed_117 = reader_117.data()
# Pandas uses np.nan as missing value.
# Thus, all columns will be of type float, regardless of their name.
expected = DataFrame(
[(np.nan, np.nan, np.nan, np.nan, np.nan)],
columns=["float_miss", "double_miss", "byte_miss", "int_miss", "long_miss"],
)
# this is an oddity as really the nan should be float64, but
# the casting doesn't fail so need to match stata here
expected["float_miss"] = expected["float_miss"].astype(np.float32)
tm.assert_frame_equal(parsed_114, expected)
tm.assert_frame_equal(parsed_117, expected)
from setup_prediction_lag import predict_abc
from load_data import extrap, abcd
from paths import paths
#----------------------------------------------------------------
seed = 1234
aux_draw = 99
#----------------------------------------------------------------
# bring in file with indexes for extrapolation bootstrap
reader = StataReader(paths.psid_bsid)
psid = reader.data(convert_dates=False, convert_categoricals=False)
psid = psid.iloc[:,0:aux_draw] # limit PSID to the number of repetitions you need
nlsy = pd.read_csv(paths.nlsy_bsid)
# set up extrapolation indexes (there are multiple data sets)
extrap_index = pd.concat([psid, nlsy], axis=0, keys=('psid', 'nlsy'), names=('dataset','id'))
extrap_source= ['psid' for j in range(0, psid.shape[0])] + ['nlsy' for k in range(0, nlsy.shape[0])]
#----------------------------------------------------------------
def boot_predict_aux(extrap, adraw):
# prepare indexes of extrapolation data for bootstrap
extrap_draw = extrap_index.loc[:, 'draw{}'.format(adraw)]
extrap_tuples = list(zip(*[extrap_source,extrap_draw]))
if not os.path.exists(paths.data):
os.mkdir(paths.data)
'''Load and Cache Datasets
-----------------------
Notes:
- Ensures no overlap in id
- Trims observations with any labor income over $300,000 (U.S., 2014)
'''
#--------------------------------------------------------------------
print "Loading PSID"
reader = StataReader(paths.psid)
psid = reader.read(convert_dates=False, convert_categoricals=False)
psid = psid.dropna(subset=['id']).set_index('id')
# Trimming
inc = psid.filter(regex='^inc_labor[0-9][0-9]')
psid = psid.loc[psid.male == 0]
psid = psid.loc[psid.black == 1]
psid = psid.loc[((inc < inc.quantile(0.90)) | (inc.isnull())).all(axis=1)]
# Interpolating
plong = pd.wide_to_long(psid[inc.columns].reset_index(),
['inc_labor'], i='id', j='age').sort_index()
plong = plong.interpolate(limit=5)
pwide = plong.unstack()
pwide.columns = pwide.columns.droplevel(0)
def meta_labels(self):
"""Read the labels for the variables and code values for the variables, using the
Stata reader. """
import re
import os
import struct
import pandas as pd
from pandas.io.stata import StataReader
var_labels = None
val_labels = None
if not os.path.exists(self.filesystem.path('meta','variable_labels.yaml')):
for name, fn in self.sources():
if name.endswith('l'):
self.log("Getting labels for {} from {} (This is really slow)".format(name, fn))
reader = StataReader(fn)
df = reader.data() # Can't get labels before reading data
var_labels = reader.variable_labels()
val_labels = reader.value_labels()
break
self.filesystem.write_yaml(var_labels, 'meta','variable_labels.yaml')
self.filesystem.write_yaml(val_labels, 'meta','value_labels.yaml')
else:
self.log("Skipping extracts; already exist")
# The value codes include both the value codes and the imputation codes. The imputation codes
# are extracted as positive integers, when they really should be negative.
table_values = {}
imputation_values = {}
if not val_labels:
val_labels = self.filesystem.read_yaml('meta','value_labels.yaml')
for k,v in val_labels.items():
table_values[k] = {}
imputation_values[k] = { -10: 'NO IMPUTATION' }
for code, code_val in v.items():
signed_code = struct.unpack('i',struct.pack('I',int(code)))[0] # Convert the unsigned to signed
if signed_code < 0:
imputation_values[k][signed_code] = code_val
else:
table_values[k][code] = code_val
self.filesystem.write_yaml(table_values, 'meta','table_codes.yaml')
self.filesystem.write_yaml(imputation_values, 'meta','imputation_codes.yaml')
self.log("{} table variables".format(len(table_values)))
self.log("{} imputation variables".format(len(imputation_values)))
return True
Desc: This code selects the IPW variables for specific ABC outcomes.
We only do this for the pooled sample to increase power. We use
a linear probiaility model for this. We select the 3 variables
that minimize the BIC.
"""
import pandas as pd
from pandas.io.stata import StataReader
import numpy as np
import statsmodels.api as sm
from patsy import dmatrices
import itertools
from paths import paths
# import data
reader = StataReader(paths.abccare)
data = reader.read(convert_dates=False, convert_categoricals=False)
data = data.set_index('id')
data = data.sort_index()
data.drop(data.loc[(data.RV==1) & (data.R==0)].index, inplace=True)
# bring in outcomes files, and find the ABC-only/CARE-only ones
outcomes = pd.read_csv(paths.outcomes, index_col='variable')
only_abc = outcomes.loc[outcomes.only_abc == 1].index
only_care = outcomes.loc[outcomes.only_care == 1].index
bank = pd.read_csv(paths.controls)
ipwvars = np.unique(outcomes.loc[~outcomes.ipw_var.isnull(),'ipw_var'].get_values())
# generate the list of all possible models
models = itertools.chain.from_iterable([itertools.combinations(bank.loc[:, 'variable'], 3)])
if not os.path.exists(paths.data):
os.mkdir(paths.data)
'''Load and Cache Datasets
-----------------------
Notes:
- Ensures no overlap in id
- Trims observations with any labor income over $300,000 (U.S., 2014)
'''
#--------------------------------------------------------------------
print "Loading CNLSY"
reader = StataReader(paths.cnlsy)
cnlsy = reader.read(convert_dates=False, convert_categoricals=False)
cnlsy = cnlsy.dropna(subset=['id']).set_index('id')
# ABC weights
wtabc = cnlsy.filter(regex='^wtabc_id[0-9]')
# Trimming
inc = cnlsy.filter(regex='^inc_labor[0-9][0-9]')
cnlsy = cnlsy.loc[((inc < 300000) | (inc.isnull())).all(axis=1)]
# Interpolating
clong = pd.wide_to_long(cnlsy[inc.columns].reset_index(),
['inc_labor'], i='id', j='age').sort_index()
clong = clong.interpolate(limit=1)
cwide = clong.unstack()
