def parse_tables():
files = fileutils.getcachecontents("io-annual")
for filename in files:
path = fileutils.getcache(filename, "io-annual")
print(path)
table = CSVTable(path, False)
make_year = is_make(filename)
use_year = is_use(filename)
if make_year:
table.create_sql_table(
"%s.annual_make_%s" % (config.IO_SCHEMA, make_year),
["industry", "commodity", "value"],
["varchar(6)", "varchar(6)", "float"])
elif use_year:
table.create_sql_table(
"%s.annual_use_%s" % (config.IO_SCHEMA, use_year),
["commodity", "industry", "value"],
["varchar(6)", "varchar(6)", "float"])
elif filename == "codes.csv":
table.create_sql_table(
"%s.annual_codes" % config.IO_SCHEMA,
["code", "description"],
["varchar(6)", "text"])
else:
continue
table.parse_to_sql()
def parse_env():
for (tablecode, tablespec) in eea_tables.items():
filename = "%s-eng.csv" % tablecode
filepath = fileutils.getcache(filename, "ca")
csvtable = CSVTable(filepath, True)
tablename = "%s.%s" % (config.SCHEMA, tablespec["tablename"])
csvtable.create_sql_table(tablename,
["year", "industry", "value"],
["int", "varchar(255)", "float"])
col_funcs = {"industry": get_industry_code}
col_map = tablespec["col_map"]
skip_callback = None
if "skip_callback" in tablespec:
skip_callback = tablespec["skip_callback"]
csvtable.parse_to_sql(col_map, col_funcs, skip_callback)
def parse_io():
# we'll just parse the same file twice,
# once each for make/use
for (agglevel, (intermediate, finaldemand)) in io_tables.items():
colnames = ["year", "industry", "commodity", "value"]
if agglevel == "detail":
coltypes = ["int", "varchar(15)", "varchar(15)", "float"]
colfuncs = {
"industry": get_industry_code,
"commodity": get_commodity_code,
}
else:
coltypes = ["int", "varchar(255)", "varchar(255)", "float"]
colfuncs = {
"industry": strip_millions, "commodity": strip_millions}
# parse intermediate
filename = "%s-eng.csv" % intermediate
filepath = fileutils.getcache(filename, "ca")
io_col_map["industry"] = "IND"
if agglevel == "detail":
io_col_map["commodity"] = "COMMOD"
else:
io_col_map["commodity"] = "COMM"
csvtable = CSVTable(filepath, True, "cp1252")
tablename = "%s.io_make_%s" % (config.SCHEMA, agglevel)
csvtable.create_sql_table(tablename, colnames, coltypes)
csvtable.parse_to_sql(io_col_map, colfuncs, skip_make, cascade=True)
# we can reuse CSVTable for the same source file
tablename = "%s.io_use_%s" % (config.SCHEMA, agglevel)
csvtable.create_sql_table(tablename, colnames, coltypes)
csvtable.parse_to_sql(io_col_map, colfuncs, skip_use)
# parse final demand
filename = "%s-eng.csv" % finaldemand
filepath = fileutils.getcache(filename, "ca")
io_col_map["commodity"] = "COMM"
io_col_map["industry"] = "CAT"
fdtable = CSVTable(filepath, True, "cp1252")
tablename = "%s.io_fd_%s" % (config.SCHEMA, agglevel)
fdtable.create_sql_table(tablename, colnames, coltypes)
if agglevel == "detail":
colfuncs["industry"] = get_fd_industry_code
fdtable.parse_to_sql(io_col_map, colfuncs, skip_finaldemand)
