def load_tiger_data():
""""""
transformation = check_epsg_for_transformation()
# if the foreign key flag is set to true reflect geoheader tables
# in matching acs schemas, tiger data is matched to acs that is one
# year less recent because it is released one year sooner
if gv.foreign_key:
acs_year = gv.tiger_year - 1
for i in ACS_SPANS:
# FIXME should probably template this on global level
acs_schema = ACS_SCHEMA.format(yr=acs_year, span=i)
try:
gv.metadata.reflect(schema=acs_schema, only=[GEOHEADER])
except sqlalchemy.exc.InvalidRequestError:
pass
for shp_path, product in gv.shp.items():
with fiona.open(shp_path) as tiger_shape:
shp_metadata = tiger_shape.meta.copy()
table = create_tiger_table(shp_metadata, product)
print '\nloading shapefile "{0}" ' \
'into table: "{1}.{2}":'.format(
basename(shp_path), gv.metadata.schema, table.name)
print 'features inserted:'
memory_tbl = list()
max_fid = max(tiger_shape.keys())
for fid, feat in tiger_shape.items():
fields = feat['properties']
row = {k.lower(): v for k, v in fields.items()}
# casting to multipolygon here because a few features
# are multi's and the geometry types must match
shapely_geom = MultiPolygon([shape(feat['geometry'])])
if transformation:
shapely_geom = ops.transform(transformation, shapely_geom)
# geoalchemy2 requires that geometry be in EWKT format
# for inserts, that conversion is made below
ga2_geom = WKTElement(shapely_geom.wkt, gv.epsg)
row['geom'] = ga2_geom
memory_tbl.append(row)
count = fid + 1
if count % 1000 == 0 or fid == max_fid:
gv.engine.execute(table.insert(), memory_tbl)
memory_tbl = list()
# logging to inform the user
if count % 20000 == 0:
sys.stdout.write(str(count))
elif fid == max_fid:
print '\n'
else:
sys.stdout.write('..')
def load_tiger_data():
""""""
transformation = check_epsg_for_transformation()
# if the foreign key flag is set to true reflect geoheader tables
# in matching acs schemas, tiger data is matched to acs that is one
# year less recent because it is released one year sooner
if gv.foreign_key:
acs_year = gv.tiger_year - 1
for i in ACS_SPANS:
# FIXME should probably template this on global level
acs_schema = ACS_SCHEMA.format(yr=acs_year, span=i)
try:
gv.metadata.reflect(schema=acs_schema, only=[GEOHEADER])
except sqlalchemy.exc.InvalidRequestError:
pass
for shp_path, product in gv.shp.items():
with fiona.open(shp_path) as tiger_shape:
shp_metadata = tiger_shape.meta.copy()
table = create_tiger_table(shp_metadata, product)
print '\nloading shapefile "{0}" ' \
'into table: "{1}.{2}":'.format(
basename(shp_path), gv.metadata.schema, table.name)
print 'features inserted:'
memory_tbl = list()
max_fid = max(tiger_shape.keys())
for fid, feat in tiger_shape.items():
fields = feat['properties']
row = {k.lower(): v for k, v in fields.items()}
# casting to multipolygon here because a few features
# are multi's and the geometry types must match
shapely_geom = MultiPolygon([shape(feat['geometry'])])
if transformation:
shapely_geom = ops.transform(transformation, shapely_geom)
# geoalchemy2 requires that geometry be in EWKT format
# for inserts, that conversion is made below
ga2_geom = WKTElement(shapely_geom.wkt, gv.epsg)
row['geom'] = ga2_geom
memory_tbl.append(row)
count = fid + 1
if count % 1000 == 0 or fid == max_fid:
gv.engine.execute(table.insert(), memory_tbl)
memory_tbl = list()
# logging to inform the user
if count % 20000 == 0:
sys.stdout.write(str(count))
elif fid == max_fid:
print '\n'
else:
sys.stdout.write('..')
def main():
""""""
# will be namespace for global variables
global gv
args = sys.argv[1:]
gv = process_options(args)
pg_url = PG_URL.format(user=gv.user, pw=gv.password,
host=gv.host, db=gv.dbname)
gv.engine = create_engine(pg_url)
gv.lookup_file = 'ACS_{span}yr_Seq_Table_Number_' \
'Lookup.txt'.format(span=gv.span)
gv.metadata = MetaData(
bind=gv.engine,
schema=ACS_SCHEMA.format(yr=gv.acs_year, span=gv.span))
download_acs_data()
drop_create_acs_schema(True)
create_geoheader()
create_acs_tables()
if gv.model:
utils.generate_model(gv.metadata, make_table_mapping(), [GEOHEADER])
