Esempio n. 1
0
	def read_stata(self, *args, **kwargs):
		reader = StataReader(*args, **kwargs)
		self.df = reader.data()
		self.variable_labels = reader.variable_labels()
		self._initialize_variable_labels()
		self.value_labels = reader.value_labels()
		# self.data_label = reader.data_label()
		return self.df
Esempio n. 2
0
    def test_read_dta1(self):
        reader = StataReader(self.dta1)
        parsed = reader.data()
        reader_13 = StataReader(self.dta1_13)
        parsed_13 = reader_13.data()
        # Pandas uses np.nan as missing value.
        # Thus, all columns will be of type float, regardless of their name.
        expected = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)],
                             columns=['float_miss', 'double_miss', 'byte_miss',
                                      'int_miss', 'long_miss'])

        # this is an oddity as really the nan should be float64, but
        # the casting doesn't fail so need to match stata here
        expected['float_miss'] = expected['float_miss'].astype(np.float32)

        tm.assert_frame_equal(parsed, expected)
        tm.assert_frame_equal(parsed_13, expected)
Esempio n. 3
0
    def test_data_method(self):
        # Minimal testing of legacy data method
        reader_114 = StataReader(self.dta1_114)
        with warnings.catch_warnings(record=True) as w:
            parsed_114_data = reader_114.data()

        reader_114 = StataReader(self.dta1_114)
        parsed_114_read = reader_114.read()
        tm.assert_frame_equal(parsed_114_data, parsed_114_read)
Esempio n. 4
0
    def test_read_dta1(self):
        reader_114 = StataReader(self.dta1_114)
        parsed_114 = reader_114.data()
        reader_117 = StataReader(self.dta1_117)
        parsed_117 = reader_117.data()
        # Pandas uses np.nan as missing value.
        # Thus, all columns will be of type float, regardless of their name.
        expected = DataFrame(
            [(np.nan, np.nan, np.nan, np.nan, np.nan)],
            columns=["float_miss", "double_miss", "byte_miss", "int_miss", "long_miss"],
        )

        # this is an oddity as really the nan should be float64, but
        # the casting doesn't fail so need to match stata here
        expected["float_miss"] = expected["float_miss"].astype(np.float32)

        tm.assert_frame_equal(parsed_114, expected)
        tm.assert_frame_equal(parsed_117, expected)
Esempio n. 5
0
    def test_data_method(self):
        # Minimal testing of legacy data method
        reader_114 = StataReader(self.dta1_114)
        with warnings.catch_warnings(record=True) as w:
            parsed_114_data = reader_114.data()

        reader_114 = StataReader(self.dta1_114)
        parsed_114_read = reader_114.read()
        tm.assert_frame_equal(parsed_114_data, parsed_114_read)
Esempio n. 6
0
    def test_read_dta1(self):
        reader = StataReader(self.dta1)
        parsed = reader.data()
        # Pandas uses np.nan as missing value. Thus, all columns will be of type float, regardless of their name.
        expected = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)],
                             columns=['float_miss', 'double_miss', 'byte_miss', 'int_miss', 'long_miss'])

        for i, col in enumerate(parsed.columns):
            np.testing.assert_almost_equal(
                parsed[col],
                expected[expected.columns[i]]
            )
Esempio n. 7
0
    def test_read_dta1(self):
        reader = StataReader(self.dta1)
        parsed = reader.data()
        # Pandas uses np.nan as missing value. Thus, all columns will be of type float, regardless of their name.
        expected = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)],
                             columns=[
                                 'float_miss', 'double_miss', 'byte_miss',
                                 'int_miss', 'long_miss'
                             ])

        for i, col in enumerate(parsed.columns):
            np.testing.assert_almost_equal(parsed[col],
                                           expected[expected.columns[i]])
Esempio n. 8
0
def _retrieve_data(dtafile):
	'''retrieve data dictionary from STATA .dta file'''

	datafile = os.path.basename(dtafile).split('.')
	if len(datafile) != 2:
		raise ValueError('dtafile must look like "file.dta"')
	if datafile[1] != 'dta':
		raise ValueError('dtafile must have ".dta" extension')

	base    = datafile[0]
	hdf     = os.path.join('.data_cache', '{}.h5'.format(base))
	lPickle = os.path.join('.data_cache', '{}_labels.pickle'.format(base))
	vPickle = os.path.join('.data_cache', '{}_vlabels.pickle'.format(base))
	dTime   = os.path.join('.data_cache', '{}_dtime.pickle'.format(base))

	if all([os.path.isfile(d) for d in [hdf, lPickle, vPickle, dTime]]):

		if os.path.getmtime(dtafile) == cPickle.load(open(dTime, 'rb')):
			from pandas import read_hdf

			data = read_hdf(hdf, 'data')
			labels = cPickle.load(open(lPickle, 'rb'))
			vlabels = cPickle.load(open(vPickle, 'rb'))

	elif not os.path.isdir('.data_cache'):
		os.makedirs('.data_cache')

	try:
		data
	except:
		from pandas.io.stata import StataReader
		from pandas import HDFStore

		print "Data is changed or no cached data found"
		print "Creating data objects from {}".format(dtafile)

		reader = StataReader(dtafile)
		data = reader.data(convert_dates=False,convert_categoricals=False)
		labels = reader.variable_labels()
		vlabels = reader.value_labels()

		store = HDFStore(hdf)
		store['data'] = data
		cPickle.dump(labels, open(lPickle, 'wb'))
		cPickle.dump(vlabels, open(vPickle, 'wb'))
		cPickle.dump(os.path.getmtime(dtafile), open(dTime, 'wb'))

		store.close()

	return {'data':data, 'labels':labels, 'vlabels':vlabels}
Esempio n. 9
0
from pandas.io.stata import StataReader

from paths import paths

reader = StataReader(paths.abccare)
abccare = reader.data(convert_dates=False, convert_categoricals=False)
abccare.id.fillna(9999,
                  inplace=True)  # This is to include the chidl with missing ID
abccare = abccare.dropna(subset=['id']).set_index('id')
abccare = abccare.sort_index()
abccare.drop(abccare.loc[abccare.abc == 0].index, inplace=True)

#abccare.drop(abccare.loc[(abccare.RV==1) & (abccare.R==0)].index, inplace=True)

# use same variable for income between CARE and ABC
#abccare.loc[abccare.program==0, 'p_inc0'] = abccare.loc[abccare.program==0, 'hh_inc0']
from setup_prediction_lag import predict_abc
from load_data import extrap, abcd

from paths import paths

#----------------------------------------------------------------

seed = 1234
aux_draw = 99

#----------------------------------------------------------------


# bring in file with indexes for extrapolation bootstrap
reader = StataReader(paths.psid_bsid)
psid = reader.data(convert_dates=False, convert_categoricals=False)
psid = psid.iloc[:,0:aux_draw] # limit PSID to the number of repetitions you need
nlsy = pd.read_csv(paths.nlsy_bsid)

# set up extrapolation indexes (there are multiple data sets)
extrap_index = pd.concat([psid, nlsy], axis=0, keys=('psid', 'nlsy'), names=('dataset','id'))
extrap_source= ['psid' for j in range(0, psid.shape[0])] + ['nlsy' for k in range(0, nlsy.shape[0])]

#----------------------------------------------------------------

def boot_predict_aux(extrap, adraw):

	# prepare indexes of extrapolation data for bootstrap
	extrap_draw = extrap_index.loc[:, 'draw{}'.format(adraw)]
	extrap_tuples = list(zip(*[extrap_source,extrap_draw]))
	for i in xrange(len(extrap_tuples) - 1, -1, -1):
Esempio n. 11
0
    def meta_labels(self):
        """Read the labels for the variables and code values for the variables, using the 
        Stata reader. """
        import re
        import os
        import struct
        import pandas as pd

        from pandas.io.stata import StataReader

        var_labels = None
        val_labels = None

        if not os.path.exists(
                self.filesystem.path('meta', 'variable_labels.yaml')):

            for name, fn in self.sources():

                if name.endswith('l'):

                    self.log(
                        "Getting labels for {}  from {} (This is really slow)".
                        format(name, fn))

                    reader = StataReader(fn)

                    df = reader.data()  # Can't get labels before reading data

                    var_labels = reader.variable_labels()
                    val_labels = reader.value_labels()

                    break

            self.filesystem.write_yaml(var_labels, 'meta',
                                       'variable_labels.yaml')
            self.filesystem.write_yaml(val_labels, 'meta', 'value_labels.yaml')

        else:
            self.log("Skipping extracts; already exist")

        # The value codes include both the value codes and the imputation codes. The imputation codes
        # are extracted  as positive integers, when they really should be negative.
        table_values = {}
        imputation_values = {}

        if not val_labels:
            val_labels = self.filesystem.read_yaml('meta', 'value_labels.yaml')

        for k, v in val_labels.items():
            table_values[k] = {}
            imputation_values[k] = {-10: 'NO IMPUTATION'}

            for code, code_val in v.items():

                signed_code = struct.unpack('i', struct.pack(
                    'I', int(code)))[0]  # Convert the unsigned to signed

                if signed_code < 0:
                    imputation_values[k][signed_code] = code_val
                else:
                    table_values[k][code] = code_val

        self.filesystem.write_yaml(table_values, 'meta', 'table_codes.yaml')
        self.filesystem.write_yaml(imputation_values, 'meta',
                                   'imputation_codes.yaml')

        self.log("{} table variables".format(len(table_values)))
        self.log("{} imputation variables".format(len(imputation_values)))

        return True
Esempio n. 12
0
from paths import paths

#----------------------------------------------------------------

seed = 1234
aux_draw = 2  # need to use more than 1
pset_type = 1

#----------------------------------------------------------------

# bring in file with indexes for interpolation bootstrap
interp_index = pd.read_csv(paths.cnlsy_bsid)

# bring in file with indexes for extrapolation bootstrap
reader = StataReader(paths.psid_bsid)
psid = reader.data(convert_dates=False, convert_categoricals=False)
psid = psid.iloc[:, 0:
                 aux_draw]  # limit PSID to the number of repetitions you need
nlsy = pd.read_csv(paths.nlsy_bsid)

# set up extrapolation indexes (there are multiple data sets)
extrap_index = pd.concat([psid, nlsy],
                         axis=0,
                         keys=('psid', 'nlsy'),
                         names=('dataset', 'id'))
extrap_source = ['psid' for j in range(0, psid.shape[0])
                 ] + ['nlsy' for k in range(0, nlsy.shape[0])]

# bring in files with weights
reader = StataReader(paths.nlsy_weights)
nlsy_weights = reader.data(convert_dates=False, convert_categoricals=False)
Esempio n. 13
0
    def meta_labels(self):
        """Read the labels for the variables and code values for the variables, using the 
        Stata reader. """
        import re
        import os
        import struct
        import pandas as pd

        from pandas.io.stata import StataReader
   
        var_labels = None
        val_labels = None

        if not os.path.exists(self.filesystem.path('meta','variable_labels.yaml')):

            for name, fn in self.sources():
   
                if name.endswith('l'):

                    self.log("Getting labels for {}  from {} (This is really slow)".format(name, fn))
   
                    reader = StataReader(fn)

                    df = reader.data() # Can't get labels before reading data
            
                    var_labels = reader.variable_labels()
                    val_labels = reader.value_labels()
                    
                    break
                    
                    
            self.filesystem.write_yaml(var_labels, 'meta','variable_labels.yaml')
            self.filesystem.write_yaml(val_labels, 'meta','value_labels.yaml')
            
        else:
            self.log("Skipping extracts; already exist")

        # The value codes include both the value codes and the imputation codes. The imputation codes
        # are extracted  as positive integers, when they really should be negative. 
        table_values = {}
        imputation_values = {}
        
        if not val_labels:
            val_labels = self.filesystem.read_yaml('meta','value_labels.yaml')
            
        for k,v in val_labels.items():
            table_values[k] = {}
            imputation_values[k] = { -10:  'NO IMPUTATION' }
        
            for code, code_val in v.items():
                
                signed_code = struct.unpack('i',struct.pack('I',int(code)))[0] # Convert the unsigned to signed
                
                if signed_code < 0:
                    imputation_values[k][signed_code] = code_val
                else:
                    table_values[k][code] = code_val

        self.filesystem.write_yaml(table_values, 'meta','table_codes.yaml')
        self.filesystem.write_yaml(imputation_values, 'meta','imputation_codes.yaml')
            
        self.log("{} table variables".format(len(table_values)))
        self.log("{} imputation variables".format(len(imputation_values)))

        return True
Esempio n. 14
0
def stataLoad(dta_filename):
    reader = StataReader(dta_filename)
    data = reader.data()
    print("\nLoaded {} rows".format(len(data)))
    return data