def read_stata(self, *args, **kwargs): reader = StataReader(*args, **kwargs) self.df = reader.data() self.variable_labels = reader.variable_labels() self._initialize_variable_labels() self.value_labels = reader.value_labels() # self.data_label = reader.data_label() return self.df
def read_stata_file(dir, file_name):
"""
:param dir: stata文件存放目录
:param file_name:
:return:返回DataFrame格式和特征表
"""
stata_data = StataReader(dir + file_name, convert_categoricals=False)
columns_list = list(stata_data.value_labels().keys()) # 列
print(file_name)
print(len(columns_list))
print(columns_list[0:10])
print('---------------')
return pd.DataFrame(stata_data.read()), columns_list
def _retrieve_data(dtafile):
'''retrieve data dictionary from STATA .dta file'''
datafile = os.path.basename(dtafile).split('.')
if len(datafile) != 2:
raise ValueError('dtafile must look like "file.dta"')
if datafile[1] != 'dta':
raise ValueError('dtafile must have ".dta" extension')
base = datafile[0]
hdf = os.path.join('.data_cache', '{}.h5'.format(base))
lPickle = os.path.join('.data_cache', '{}_labels.pickle'.format(base))
vPickle = os.path.join('.data_cache', '{}_vlabels.pickle'.format(base))
dTime = os.path.join('.data_cache', '{}_dtime.pickle'.format(base))
if all([os.path.isfile(d) for d in [hdf, lPickle, vPickle, dTime]]):
if os.path.getmtime(dtafile) == cPickle.load(open(dTime, 'rb')):
from pandas import read_hdf
data = read_hdf(hdf, 'data')
labels = cPickle.load(open(lPickle, 'rb'))
vlabels = cPickle.load(open(vPickle, 'rb'))
elif not os.path.isdir('.data_cache'):
os.makedirs('.data_cache')
try:
data
except:
from pandas.io.stata import StataReader
from pandas import HDFStore
print "Data is changed or no cached data found"
print "Creating data objects from {}".format(dtafile)
reader = StataReader(dtafile)
data = reader.data(convert_dates=False,convert_categoricals=False)
labels = reader.variable_labels()
vlabels = reader.value_labels()
store = HDFStore(hdf)
store['data'] = data
cPickle.dump(labels, open(lPickle, 'wb'))
cPickle.dump(vlabels, open(vPickle, 'wb'))
cPickle.dump(os.path.getmtime(dtafile), open(dTime, 'wb'))
store.close()
return {'data':data, 'labels':labels, 'vlabels':vlabels}
def meta_labels(self):
"""Read the labels for the variables and code values for the variables, using the
Stata reader. """
import re
import os
import struct
import pandas as pd
from pandas.io.stata import StataReader
var_labels = None
val_labels = None
if not os.path.exists(
self.filesystem.path('meta', 'variable_labels.yaml')):
for name, fn in self.sources():
if name.endswith('l'):
self.log(
"Getting labels for {} from {} (This is really slow)".
format(name, fn))
reader = StataReader(fn)
df = reader.data() # Can't get labels before reading data
var_labels = reader.variable_labels()
val_labels = reader.value_labels()
break
self.filesystem.write_yaml(var_labels, 'meta',
'variable_labels.yaml')
self.filesystem.write_yaml(val_labels, 'meta', 'value_labels.yaml')
else:
self.log("Skipping extracts; already exist")
# The value codes include both the value codes and the imputation codes. The imputation codes
# are extracted as positive integers, when they really should be negative.
table_values = {}
imputation_values = {}
if not val_labels:
val_labels = self.filesystem.read_yaml('meta', 'value_labels.yaml')
for k, v in val_labels.items():
table_values[k] = {}
imputation_values[k] = {-10: 'NO IMPUTATION'}
for code, code_val in v.items():
signed_code = struct.unpack('i', struct.pack(
'I', int(code)))[0] # Convert the unsigned to signed
if signed_code < 0:
imputation_values[k][signed_code] = code_val
else:
table_values[k][code] = code_val
self.filesystem.write_yaml(table_values, 'meta', 'table_codes.yaml')
self.filesystem.write_yaml(imputation_values, 'meta',
'imputation_codes.yaml')
self.log("{} table variables".format(len(table_values)))
self.log("{} imputation variables".format(len(imputation_values)))
return True
def meta_labels(self):
"""Read the labels for the variables and code values for the variables, using the
Stata reader. """
import re
import os
import struct
import pandas as pd
from pandas.io.stata import StataReader
var_labels = None
val_labels = None
if not os.path.exists(self.filesystem.path('meta','variable_labels.yaml')):
for name, fn in self.sources():
if name.endswith('l'):
self.log("Getting labels for {} from {} (This is really slow)".format(name, fn))
reader = StataReader(fn)
df = reader.data() # Can't get labels before reading data
var_labels = reader.variable_labels()
val_labels = reader.value_labels()
break
self.filesystem.write_yaml(var_labels, 'meta','variable_labels.yaml')
self.filesystem.write_yaml(val_labels, 'meta','value_labels.yaml')
else:
self.log("Skipping extracts; already exist")
# The value codes include both the value codes and the imputation codes. The imputation codes
# are extracted as positive integers, when they really should be negative.
table_values = {}
imputation_values = {}
if not val_labels:
val_labels = self.filesystem.read_yaml('meta','value_labels.yaml')
for k,v in val_labels.items():
table_values[k] = {}
imputation_values[k] = { -10: 'NO IMPUTATION' }
for code, code_val in v.items():
signed_code = struct.unpack('i',struct.pack('I',int(code)))[0] # Convert the unsigned to signed
if signed_code < 0:
imputation_values[k][signed_code] = code_val
else:
table_values[k][code] = code_val
self.filesystem.write_yaml(table_values, 'meta','table_codes.yaml')
self.filesystem.write_yaml(imputation_values, 'meta','imputation_codes.yaml')
self.log("{} table variables".format(len(table_values)))
self.log("{} imputation variables".format(len(imputation_values)))
return True
