def test_tablemeta(self):
tm = TableMeta(2016, 1)
tm._process()
print(len(tm.tables))
print(list(tm.tables.keys())[500:550])
self.assertEqual(
tm.tables['C16004'].title,
'AGE BY LANGUAGE SPOKEN AT HOME BY ABILITY TO SPEAK ENGLISH FOR THE POPULATION 5 YEARS AND OVER'
)
with open('/tmp/tables_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(Table.csv_header)
w.writerows(t.row for t in tm.tables.values())
with open('/tmp/columns_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(Column.csv_header)
for t in tm.tables.values():
for cn in sorted(t.columns):
c = t.columns[cn]
w.writerow(c.row)
def test_tablemeta(self):
tm = TableMeta(2016, 1)
tm._process()
self.assertEqual(1310, len(tm.tables))
self.assertEqual(sorted(['b00001', 'b00002', 'b01001', 'b01001a', 'b01001b', 'b01001c',
'b01001d', 'b01001e', 'b01001f', 'b01001g']),
sorted(list(tm.tables.keys())[:10]))
self.assertEqual(sorted(['b15011', 'c15002d', 'c15002e', 'c15002f', 'c15002g',
'c15002h', 'c15002i', 'c15003', 'c15010', 'c15010a']),
sorted(list(tm.tables.keys())[500:510]))
self.assertEqual(tm.tables['c16004'].title,
'Age By Language Spoken At Home By Ability To Speak English For The Population 5 Years And Over')
with open('/tmp/tables_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(Table.csv_header)
w.writerows(t.row for t in tm.tables.values())
with open('/tmp/columns_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(Column.csv_header)
for t in tm.tables.values():
for cn in sorted(t.columns):
c = t.columns[cn]
w.writerow(c.row )
def test_age_dimensions(self):
'''Check that there are not tables with 'year' in the title that don't get a parsed age range'''
tm = TableMeta(2016, 5)
age_tables = []
for t_id, table in tm.tables.items():
if 'by age' in table.title.lower():
age_tables.append(t_id)
for at in age_tables:
u = parse_app_url('census://2016/5/RI/40/{}'.format(at.lower()))
g = u.generator
t = g.table
parse_errors = []
for c in t.columns:
if '_m90' not in c.unique_id and 'year' in c.description and not c.age_range and '1 year ago' not in \
c.description and 'year-round' not in c.description:
parse_errors.append(c)
for parse_error in parse_errors:
print(parse_error.row)
self.assertEqual(0, len(parse_errors))
def test_pov_dimensions(self):
tm = TableMeta(2016, 5)
for t_id, table in tm.tables.items():
u = parse_app_url('census://2016/5/RI/40/{}'.format(t_id.lower()))
g = u.generator
t = g.table
for i, c in enumerate(t.columns):
if '_m90' not in c.unique_id and 'poverty' in c.long_description:
print(c.unique_id, c.poverty_status, c.long_description)
def test_tablemeta(self):
tm = TableMeta(2016, 1)
tm._process()
self.assertEqual(1310, len(tm.tables))
keys = [
'b18135', 'b18140', 'b19001', 'b19001a', 'b19001b', 'b19001c',
'b19001d', 'b19001e', 'b19001f', 'b19001g', 'b19001h', 'b19001i',
'b19013', 'b19013a', 'b19013b', 'b19013c', 'b19013d', 'b19013e',
'b19013f', 'b19013g', 'b19013h', 'b19013i', 'b19019', 'b19025',
'b19025a', 'b19025b', 'b19025c', 'b19025d', 'b19025e', 'b19025f',
'b19025g', 'b19025h', 'b19025i', 'b19037', 'b19037a', 'b19037b',
'b19037c', 'b19037d', 'b19037e', 'b19037f', 'b19037g', 'b19037h',
'b19037i', 'b19049', 'b19050', 'b19051', 'b19052', 'b19053',
'b19054', 'b19055'
]
self.assertEqual(keys, sorted(tm.tables.keys())[500:550])
self.assertEqual(
tm.tables['c16004'].title.upper(),
'AGE BY LANGUAGE SPOKEN AT HOME BY ABILITY TO SPEAK ENGLISH FOR THE POPULATION 5 YEARS AND OVER'
)
with open('/tmp/tables_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(MetaTable.csv_header)
w.writerows(t.row for t in tm.tables.values())
with open('/tmp/columns_meta.csv', 'w') as f:
w = csv.writer(f)
w.writerow(Column.csv_header)
for t in tm.tables.values():
for cn in sorted(t.columns):
c = t.columns[cn]
w.writerow(c.row)
def __init__(self, ref, cache=None, working_dir=None, **kwargs):
from geoid.acs import AcsGeoid
from publicdata.census.files.metafiles import TableMeta
super().__init__(ref, cache, working_dir, **kwargs)
gid = AcsGeoid.parse(self.ref.geoid)
self.table = Table(self.ref.year, self.ref.release, gid.stusab,
str(self.ref.summary_level), self.ref.tableid)
self._meta = TableMeta(self.ref.year, self.ref.release)
assert isinstance(ref, CensusUrl)
def test_dimensions(self):
tm = TableMeta(2016, 5)
with open('/tmp/dimensions.csv', 'w') as f:
w = csv.writer(f)
w.writerow("id sex race age pov description".split())
for t_id, table in tm.tables.items():
u = parse_app_url('census://2016/5/RI/40/{}'.format(t_id.lower()))
g = u.generator
t = g.table
w.writerow([t_id, table.title])
for i, c in enumerate(t.columns):
if '_m90' not in c.unique_id and i > 3:
row = [c.unique_id, c.sex, c.race, c.age, c.poverty_status, c.description]
print(row)
w.writerow(row)
def __init__(self, ref, cache=None, working_dir=None, **kwargs):
from geoid.acs import AcsGeoid
from publicdata.census.files.metafiles import TableMeta
super().__init__(ref, cache, working_dir, **kwargs)
gid = AcsGeoid.parse(self.ref.geoid)
try:
stusab = gid.stusab
except AttributeError:
gid.us# "US" level geoids don't have a state parameter.
stusab = "US"
self.table = Table(self.ref.year, self.ref.release, stusab,
str(self.ref.summary_level), self.ref.tableid)
self._meta = TableMeta(self.ref.year, self.ref.release)
self._source_url = kwargs.get('source_url')
assert(self._source_url)
assert isinstance(ref, CensusUrl)
def tablemeta(year, release):
"""Function to cache creation of TableMeta objects"""
return TableMeta(year, release)
def tablemeta(year, release):
return TableMeta(year, release)