def load_census_data(type):
print >>sys.stderr, "Loading census_data table..."
geoTables = {}
for row in census.parse_sum_files([type]): #,requesting_keys[type]):
(layout, logrecno, fileid, stusab, chariter, cifsn, t, geo_file) = map(row.pop, \
['layout', 'LOGRECNO', 'FILEID', 'STUSAB', 'CHARITER', 'CIFSN', 'type', 'geo_file'])
(geo_file, geo_args) = geo_file
logrecno = int(logrecno)
if geo_file not in geoTables:
reqed_keys = set(['LOGRECNO','SUMLEV','STATE','CD110'])
tmp = census.build_geo_table(geo_file, geo_args)
geoTables[geo_file] = dict([ (lrn, dict([(k,d[k]) for k in filter(lambda x: x in reqed_keys, d.keys())])) for lrn,d in tmp.items()])
geo = geoTables[geo_file]
if logrecno in geo:
### Entries for states
loc_code = ''
if geo[logrecno]['SUMLEV'] == 'STATE':
loc_code = geo[logrecno]['STATE']
### Entries for the 110th Congress.
elif geo[logrecno]['SUMLEV'] == 'DISTS' \
and geo[logrecno]['CD110']:
loc_code = '%s-%02d' % (geo[logrecno]['STATE'], int(geo[logrecno]['CD110']))
else: continue
# Fix some districts:
if loc_code in ['DC','PR']: loc_code = loc_code+'-00'
if loc_code in ['DC-98','PR-98']: continue
for internal_key, value in row.items():
db.insert('census_data', seqname=False, district_id=loc_code, internal_key=internal_key, census_type=type, value=value)
print >>sys.stderr, "...Done loading census_data table."
def load_census_population():
print >> sys.stderr, "Loading census_population table..."
#db.delete('census_population', where='1=1')
geoTables = {}
# Load the population data from SF101.
required_geo_keys = set([
'LOGRECNO', 'SUMLEV', 'STATE', 'CD110', 'COUNTY', 'BLKGRP', 'BLOCK',
'TRACT', 'ZCTA5', 'AREALAND'
])
desired_sumlevs = set(
['STATE', 'COUNTY', 'DISTS', 'ZCTA', 'TRACT', 'BLOCK'])
for row in census.parse_state_sum_files([1], ['P002001']):
(layout, logrecno, fileid, stusab, chariter, cifsn, t,
geo_file) = map(row.pop, [
'layout', 'LOGRECNO', 'FILEID', 'STUSAB', 'CHARITER', 'CIFSN',
'type', 'geo_file'
])
(geo_file, geo_args) = geo_file
logrecno = int(logrecno)
geo = None
if geo_file not in geoTables:
geoTables = dict()
geoTables[geo_file] = dict([(
lrn,
dict([
(k, d[k])
for k in filter(lambda x: x in required_geo_keys, d.keys())
])
) for lrn, d in census.build_geo_table(geo_file, geo_args).items()
])
geo = geoTables[geo_file]
if logrecno not in geo: continue
if geo[logrecno]['SUMLEV'] in desired_sumlevs and 'P002001' in row:
if geo[logrecno]['SUMLEV'] == 'DISTS' and \
not geo[logrecno]['CD110']:
continue
#print "inserting", geo[logrecno]['SUMLEV']
db.insert('census_population',
state_id=geo[logrecno]['STATE'],
county_id=geo[logrecno]['COUNTY'],
blockgrp_id=geo[logrecno]['BLKGRP'],
block_id=geo[logrecno]['BLOCK'],
district_id=geo[logrecno]['CD110'],
zip_id=geo[logrecno]['ZCTA5'],
sumlev=geo[logrecno]['SUMLEV'],
tract_id=geo[logrecno]['TRACT'],
area_land=geo[logrecno]['AREALAND'],
population=row['P002001'])
#else: print "oops", geo[logrecno]['SUMLEV']
print >> sys.stderr, "...Done loading census_population table."
def load_census_data(type):
print >> sys.stderr, "Loading census_data table..."
geoTables = {}
for row in census.parse_sum_files([type]): #,requesting_keys[type]):
(layout, logrecno, fileid, stusab, chariter, cifsn, t, geo_file) = map(row.pop, \
['layout', 'LOGRECNO', 'FILEID', 'STUSAB', 'CHARITER', 'CIFSN', 'type', 'geo_file'])
(geo_file, geo_args) = geo_file
logrecno = int(logrecno)
if geo_file not in geoTables:
reqed_keys = set(['LOGRECNO', 'SUMLEV', 'STATE', 'CD110'])
tmp = census.build_geo_table(geo_file, geo_args)
geoTables[geo_file] = dict([
(lrn,
dict([(k, d[k])
for k in filter(lambda x: x in reqed_keys, d.keys())]))
for lrn, d in tmp.items()
])
geo = geoTables[geo_file]
if logrecno in geo:
### Entries for states
loc_code = ''
if geo[logrecno]['SUMLEV'] == 'STATE':
loc_code = geo[logrecno]['STATE']
### Entries for the 110th Congress.
elif geo[logrecno]['SUMLEV'] == 'DISTS' \
and geo[logrecno]['CD110']:
loc_code = '%s-%02d' % (geo[logrecno]['STATE'],
int(geo[logrecno]['CD110']))
else:
continue
# Fix some districts:
if loc_code in ['DC', 'PR']: loc_code = loc_code + '-00'
if loc_code in ['DC-98', 'PR-98']: continue
for internal_key, value in row.items():
db.insert('census_data',
seqname=False,
district_id=loc_code,
internal_key=internal_key,
census_type=type,
value=value)
print >> sys.stderr, "...Done loading census_data table."
def load_census_population():
print >>sys.stderr, "Loading census_population table..."
#db.delete('census_population', where='1=1')
geoTables = {}
# Load the population data from SF101.
required_geo_keys = set(['LOGRECNO', 'SUMLEV', 'STATE', 'CD110', 'COUNTY',
'BLKGRP', 'BLOCK', 'TRACT', 'ZCTA5', 'AREALAND'])
desired_sumlevs = set(['STATE', 'COUNTY', 'DISTS', 'ZCTA', 'TRACT',
'BLOCK'])
for row in census.parse_state_sum_files([1], ['P002001']):
(layout, logrecno, fileid, stusab, chariter, cifsn, t, geo_file) = map(row.pop,
['layout', 'LOGRECNO', 'FILEID', 'STUSAB', 'CHARITER', 'CIFSN', 'type', 'geo_file'])
(geo_file, geo_args) = geo_file
logrecno = int(logrecno)
geo = None
if geo_file not in geoTables:
geoTables = dict()
geoTables[geo_file] = dict([ (lrn, dict([(k,d[k]) for k in filter(lambda x: x in required_geo_keys, d.keys())])) for lrn,d in census.build_geo_table(geo_file, geo_args).items() ])
geo = geoTables[geo_file]
if logrecno not in geo: continue
if geo[logrecno]['SUMLEV'] in desired_sumlevs and 'P002001' in row:
if geo[logrecno]['SUMLEV'] == 'DISTS' and \
not geo[logrecno]['CD110']:
continue
#print "inserting", geo[logrecno]['SUMLEV']
db.insert('census_population',
state_id = geo[logrecno]['STATE'],
county_id = geo[logrecno]['COUNTY'],
blockgrp_id = geo[logrecno]['BLKGRP'],
block_id = geo[logrecno]['BLOCK'],
district_id = geo[logrecno]['CD110'],
zip_id = geo[logrecno]['ZCTA5'],
sumlev = geo[logrecno]['SUMLEV'],
tract_id = geo[logrecno]['TRACT'],
area_land = geo[logrecno]['AREALAND'],
population = row['P002001'])
#else: print "oops", geo[logrecno]['SUMLEV']
print >>sys.stderr, "...Done loading census_population table."