def test_read_dta18(self): parsed_118 = self.read_dta(self.dta22_118) parsed_118["Bytes"] = parsed_118["Bytes"].astype('O') expected = DataFrame.from_records( [['Cat', 'Bogota', u'Bogotá', 1, 1.0, u'option b Ünicode', 1.0], ['Dog', 'Boston', u'Uzunköprü', np.nan, np.nan, np.nan, np.nan], ['Plane', 'Rome', u'Tromsø', 0, 0.0, 'option a', 0.0], ['Potato', 'Tokyo', u'Elâzığ', -4, 4.0, 4, 4], ['', '', '', 0, 0.3332999, 'option a', 1/3.] ], columns=['Things', 'Cities', 'Unicode_Cities_Strl', 'Ints', 'Floats', 'Bytes', 'Longs']) expected["Floats"] = expected["Floats"].astype(np.float32) for col in parsed_118.columns: tm.assert_almost_equal(parsed_118[col], expected[col]) rdr = StataReader(self.dta22_118) vl = rdr.variable_labels() vl_expected = {u'Unicode_Cities_Strl': u'Here are some strls with Ünicode chars', u'Longs': u'long data', u'Things': u'Here are some things', u'Bytes': u'byte data', u'Ints': u'int data', u'Cities': u'Here are some cities', u'Floats': u'float data'} tm.assert_dict_equal(vl, vl_expected) self.assertEqual(rdr.data_label, u'This is a Ünicode data label')
def read_stata(self, *args, **kwargs): reader = StataReader(*args, **kwargs) self.df = reader.data() self.variable_labels = reader.variable_labels() self._initialize_variable_labels() self.value_labels = reader.value_labels() # self.data_label = reader.data_label() return self.df
def _retrieve_data(dtafile): '''retrieve data dictionary from STATA .dta file''' datafile = os.path.basename(dtafile).split('.') if len(datafile) != 2: raise ValueError('dtafile must look like "file.dta"') if datafile[1] != 'dta': raise ValueError('dtafile must have ".dta" extension') base = datafile[0] hdf = os.path.join('.data_cache', '{}.h5'.format(base)) lPickle = os.path.join('.data_cache', '{}_labels.pickle'.format(base)) vPickle = os.path.join('.data_cache', '{}_vlabels.pickle'.format(base)) dTime = os.path.join('.data_cache', '{}_dtime.pickle'.format(base)) if all([os.path.isfile(d) for d in [hdf, lPickle, vPickle, dTime]]): if os.path.getmtime(dtafile) == cPickle.load(open(dTime, 'rb')): from pandas import read_hdf data = read_hdf(hdf, 'data') labels = cPickle.load(open(lPickle, 'rb')) vlabels = cPickle.load(open(vPickle, 'rb')) elif not os.path.isdir('.data_cache'): os.makedirs('.data_cache') try: data except: from pandas.io.stata import StataReader from pandas import HDFStore print "Data is changed or no cached data found" print "Creating data objects from {}".format(dtafile) reader = StataReader(dtafile) data = reader.data(convert_dates=False,convert_categoricals=False) labels = reader.variable_labels() vlabels = reader.value_labels() store = HDFStore(hdf) store['data'] = data cPickle.dump(labels, open(lPickle, 'wb')) cPickle.dump(vlabels, open(vPickle, 'wb')) cPickle.dump(os.path.getmtime(dtafile), open(dTime, 'wb')) store.close() return {'data':data, 'labels':labels, 'vlabels':vlabels}
def meta_labels(self): """Read the labels for the variables and code values for the variables, using the Stata reader. """ import re import os import struct import pandas as pd from pandas.io.stata import StataReader var_labels = None val_labels = None if not os.path.exists( self.filesystem.path('meta', 'variable_labels.yaml')): for name, fn in self.sources(): if name.endswith('l'): self.log( "Getting labels for {} from {} (This is really slow)". format(name, fn)) reader = StataReader(fn) df = reader.data() # Can't get labels before reading data var_labels = reader.variable_labels() val_labels = reader.value_labels() break self.filesystem.write_yaml(var_labels, 'meta', 'variable_labels.yaml') self.filesystem.write_yaml(val_labels, 'meta', 'value_labels.yaml') else: self.log("Skipping extracts; already exist") # The value codes include both the value codes and the imputation codes. The imputation codes # are extracted as positive integers, when they really should be negative. table_values = {} imputation_values = {} if not val_labels: val_labels = self.filesystem.read_yaml('meta', 'value_labels.yaml') for k, v in val_labels.items(): table_values[k] = {} imputation_values[k] = {-10: 'NO IMPUTATION'} for code, code_val in v.items(): signed_code = struct.unpack('i', struct.pack( 'I', int(code)))[0] # Convert the unsigned to signed if signed_code < 0: imputation_values[k][signed_code] = code_val else: table_values[k][code] = code_val self.filesystem.write_yaml(table_values, 'meta', 'table_codes.yaml') self.filesystem.write_yaml(imputation_values, 'meta', 'imputation_codes.yaml') self.log("{} table variables".format(len(table_values))) self.log("{} imputation variables".format(len(imputation_values))) return True
def meta_labels(self): """Read the labels for the variables and code values for the variables, using the Stata reader. """ import re import os import struct import pandas as pd from pandas.io.stata import StataReader var_labels = None val_labels = None if not os.path.exists(self.filesystem.path('meta','variable_labels.yaml')): for name, fn in self.sources(): if name.endswith('l'): self.log("Getting labels for {} from {} (This is really slow)".format(name, fn)) reader = StataReader(fn) df = reader.data() # Can't get labels before reading data var_labels = reader.variable_labels() val_labels = reader.value_labels() break self.filesystem.write_yaml(var_labels, 'meta','variable_labels.yaml') self.filesystem.write_yaml(val_labels, 'meta','value_labels.yaml') else: self.log("Skipping extracts; already exist") # The value codes include both the value codes and the imputation codes. The imputation codes # are extracted as positive integers, when they really should be negative. table_values = {} imputation_values = {} if not val_labels: val_labels = self.filesystem.read_yaml('meta','value_labels.yaml') for k,v in val_labels.items(): table_values[k] = {} imputation_values[k] = { -10: 'NO IMPUTATION' } for code, code_val in v.items(): signed_code = struct.unpack('i',struct.pack('I',int(code)))[0] # Convert the unsigned to signed if signed_code < 0: imputation_values[k][signed_code] = code_val else: table_values[k][code] = code_val self.filesystem.write_yaml(table_values, 'meta','table_codes.yaml') self.filesystem.write_yaml(imputation_values, 'meta','imputation_codes.yaml') self.log("{} table variables".format(len(table_values))) self.log("{} imputation variables".format(len(imputation_values))) return True