def graph_table(filename, title, vector,
base_year, is_intensities, collapse=[]):
plot = GNUPlot(filename, title, "usa")
data = {}
sortby = {}
other_category = "Other"
max_year = max(config.STUDY_YEARS)
other_category_numer = dict((year, 0) for year in config.STUDY_YEARS)
other_category_denom = dict((year, 0) for year in config.STUDY_YEARS)
for group in vector.keys():
numerator = vector[group]
if is_intensities:
denominator = pce_dollars[group]
else:
denominator = dict((year, 1) for year in config.STUDY_YEARS)
base_value = numerator[base_year] / denominator[base_year]
if group in collapse:
for year in config.STUDY_YEARS:
other_category_numer[year] += numerator[year]
# adding 1 for each year is fine since it's relative to the base
other_category_denom[year] += denominator[year]
else:
data[group] = {}
for year in config.STUDY_YEARS:
data[group][year] = numerator[year] / denominator[year]
sortby[ data[group][max_year] / data[group][base_year] ] = group
if len(other_category_numer):
data[other_category] = dict(
(year, other_category_numer[year] / other_category_denom[year]) \
for year in config.STUDY_YEARS)
sortby[ data[other_category][max_year] /\
data[other_category][base_year] ] = other_category
sorted_groups = [sortby[key] for key in sorted(sortby.keys(), reverse=True)]
plot.add_custom_setup("set style data linespoints")
plot.add_custom_setup("unset colorbox")
plot.add_custom_setup("set grid")
min_x = 5 * math.floor(min(config.STUDY_YEARS) / 5)
max_x = 5 * math.ceil(max(config.STUDY_YEARS) / 5) + 5 # room for labels
plot.add_custom_setup("set xrange [ %d : %d ]" % (min_x, max_x))
interval = 1 / (len(sorted_groups) - 1)
for i in range(len(sorted_groups)):
plot.add_custom_setup("set style lines %d lc palette frac %.2f lw 2"
% (i + 1, i * interval))
series_values = []
for i in range(len(sorted_groups)):
group = sorted_groups[i]
if group in bea.short_nipa:
group_name = bea.short_nipa[group]
else:
group_name = group
series_values.append(group_name)
plot.suppress_title(group_name)
base_value = data[group][base_year]
for year in config.STUDY_YEARS:
print(group, year, data[group][year], base_value, data[group][year] / base_value)
plot.set_value(group_name, year,
data[group][year] / base_value * 100)
plot.set_series_style(group_name, "linestyle %d" % (i + 1))
plot.series_values = series_values
plot.write_tables()
plot.get_axis_specs() # make sure max_y is adjusted
prev_pos = None
for i in range(len(sorted_groups)):
group = sorted_groups[i]
if group in bea.short_nipa:
group_name = bea.short_nipa[group]
else:
group_name = group
ending_value = data[group][max_year] / data[group][base_year] * 100
position = ending_value / plot.max_y + 0.01 # line up baseline
# space labels out by at least 0.04 so they don't overlap
if prev_pos is not None and prev_pos - position < 0.03:
position = prev_pos - 0.03
plot.add_custom_setup(
"set label %d '%s' at graph 0.81, %.2f font 'Arial,8'"
% (i + 1, group_name, position))
prev_pos = position
plot.generate_plot()
content = GNUPlot.get_plotscript_content(self)
self.writing_plotscript = False
return content
def get_plot_clauses(self):
clauses = GNUPlot.get_plot_clauses(self)
column = len(self.xvalues) + 1
clauses.append(
"'' using %d axis x1y2 with linespoints lc rgb 'black' title column(%d)"
% (column, column))
return clauses
dirname = "usa-pce-energy"
ff_plot = GNUPlot("total-ff", "Fossil fuels (trillion Btu)", dirname)
elec_plot = GNUPlot("total-elec", "Electricity (trillion Btu)", dirname)
ff_iplot = GNUPlot("total-intensity-ff", "Fossil fuels (Btu per dollar)",
dirname)
elec_iplot = GNUPlot("total-intensity-elec", "Electricity (Btu per dollar)",
dirname)
ff_plot.style = "histogram"
ff_iplot.style = "histogram"
elec_plot.style = "histogram rowstacked"
elec_iplot.style = "histogram rowstacked"
for year in config.STUDY_YEARS:
# args: year, is_hybrid, allow_imports, adjust for inflation
iogen = common.iogen_for_year(year, True, True, True)
fd_dollars = {
"exports": export_n.sum(),
"pce": Yn.get_pce().sum(),
"imports": import_n.sum(),
}
###### breakdown by fd sector
dirname = "usa-pce-energy"
for (key, vector) in fd_vectors.items():
fdgroup = bea.fd_sector_names[key]
ikey = key + "-intensity"
if key not in plots:
plots[key] = GNUPlot(key, fdgroup + " (trillion Btu)", dirname)
plots[key].style = "histogram"
if ikey not in plots:
plots[ikey] = GNUPlot(ikey, fdgroup + " (MMBtu per 2005$)", dirname)
plots[ikey].style = "histogram"
use_vector = L.matrix_mult(vector)
for i in range(len(energy_codes)):
code = energy_codes[i]
name = eia.name_for_naics(code)
value = use_vector.get_element(rowname=code) # bBtu
# divide bBtu by 1000 for tBtu
plots[key].set_value(year, name, value / 1000)
# div bBtu by k$ for MMBtu/$
def trade_ratios(base_country, env_key):
env_series = config.env_series_names[env_key]
other_countries = sorted(config.countries.keys())
other_countries.remove(base_country)
yearstrings = [str(y) for y in config.STUDY_YEARS]
exported_y = NamedMatrix(rows=other_countries, cols=yearstrings)
exported_E = NamedMatrix(rows=other_countries, cols=yearstrings)
def describe_exporters():
print("Exports by country - " + env_key)
print("dollars")
for country in other_countries:
formatted = [
("%.0f" % exported_y.get_element(country, year)).rjust(6)
for year in yearstrings
]
print(country, " ".join(formatted))
print("emissions")
for country in other_countries:
formatted = [
("%.0f" % exported_E.get_element(country, year)).rjust(6)
for year in yearstrings
]
print(country, " ".join(formatted))
print("intensities")
intensities = exported_E.divide(exported_y)
for country in other_countries:
formatted = [
("%.2f" % intensities.get_element(country, year)).rjust(6)
for year in yearstrings
]
print(country, " ".join(formatted))
### prepare plots
env_title = env_key.replace(" ", "-")
plot_group = "import-%s-%s" % (env_title, base_country)
plots = {}
for sector in common.default_env_sectors:
sector_title = common.get_industry_title(sector)
plots[sector] = GNUPlot(
"%s_%s" % (env_title, sector),
"",
#"Imports of %s - %s" % (sector_title, env_key),
plot_group)
plots["import"] = GNUPlot(
"%s_import" % env_title,
"",
#"All imports - %s" % env_key,
plot_group)
plots["export"] = GNUPlot(
"%s_export" % env_title,
"",
#"All imports - %s" % env_key,
plot_group)
def create_plots():
for (sector, plot) in plots.items():
plot.set_series_style(
import_dollar_series,
"pointtype 5 linetype rgb '#88BBFF'") # hollow circle
plot.set_series_style(
emission_series,
"pointtype 7 linetype rgb '#1144FF'") # solid circle
plot.set_series_style(self_emission_series,
"pointtype 12 linetype rgb '#0000FF'")
plot.set_series_style(export_dollar_series,
"pointtype 4 linetype rgb '#CC9933'")
plot.set_series_style(export_emissions_series,
"pointtype 7 linetype rgb '#996600'")
#plot.legend("width -8")
plot.width = 480
plot.height = 300
plot.write_tables()
plot.generate_plot()
### trade balance matrices
export_balance = NamedMatrix(rows=common.default_env_sectors,
cols=yearstrings)
import_balance = NamedMatrix(rows=common.default_env_sectors,
cols=yearstrings)
all_E = NamedMatrix(rows=common.default_env_sectors, cols=yearstrings)
def describe_balance(ratios=False):
export_total = export_balance.sum(0)
import_total = import_balance.sum(0)
all_total = all_E.sum(0)
balance = export_total.subtract(import_total)
country_name = config.countries[base_country]
oddyears = [
str(y) for y in filter(lambda x: x % 2 == 1, config.STUDY_YEARS)
]
if ratios:
balance_ratio = balance.divide(all_total)
print(
country_name.ljust(15) + " & " +
" & ".join([("%.2f" %
balance_ratio.get_element("sum", y)).rjust(6)
for y in oddyears]) + " \\\\")
else:
print(
country_name.ljust(15) + " & " + " & ".join([
utils.add_commas(balance.get_element("sum", y)).rjust(9)
for y in oddyears
]) + " \\\\")
def describe_balance_intensity():
export_total = export_balance.sum(0)
import_total = import_balance.sum(0)
all_total = all_E.sum(0)
balance = export_total.subtract(import_total)
balance_ratio = balance.divide(all_total)
country_name = config.countries[base_country]
years = [str(minyear), str(maxyear)]
fields = []
for y in years:
balance_val = balance.get_element("sum", y)
ratio_val = balance_ratio.get_element("sum", y)
(gdp_val, env_val, intensity) = \
common.get_efficiency(base_country, int(y))
if int(y) in balance_plots:
plot = balance_plots[int(y)]
# ratio = exports to imports
plot.set_value("2 ratio", base_country, ratio_val)
plot.set_value("1 intensity", base_country, intensity)
if int(y) in worldmap:
worldmap[int(y)].set_country_value(base_country, balance_val)
fields.append(utils.add_commas(balance_val))
fields.append("%.2f" % ratio_val)
fields.append("%.2f" % intensity)
print(country_name.ljust(15) + " & " + " & ".join(fields) + " \\NN")
###
iogen = common.iogen_for_year(config.STUDY_YEARS[0])
envgen = common.envgen_for_year(config.STUDY_YEARS[0])
strings = {"schema": config.WIOD_SCHEMA}
for year in config.STUDY_YEARS:
strings["year"] = year
base_E = get_wiod_env_vector(base_country, year, env_series)
all_E.set_column(str(year), base_E)
base_E_sum = base_E.sum()
iogen.set_table("%(schema)s.indbyind_%(year)d" % strings)
iogen.set_condition_args(base_country)
envgen.set_table("%(schema)s.env_%(year)d" % strings)
envgen.set_condition_args(base_country)
envgen.set_sectors(common.default_env_sectors)
# prepare base country values
y = iogen.get_Y()
import_column = common.get_import_vector(iogen)
base_imports = import_column.sum()
base_gdp = y.sum()
harmonizer = common.get_io_harmonizer(iogen)
base_y = harmonizer.matrix_mult(y.get_total())
x = iogen.get_x()
E = envgen.get_env_vector(env_series)
L = iogen.get_L()
J = E.divide(harmonizer.matrix_mult(x), ignore_zero_denom=True)
base_JsL = J.square_matrix_from_diag()\
.matrix_mult(harmonizer).matrix_mult(L)
exported = base_JsL.matrix_mult(y.get_exports())
export_balance.set_column(str(year), exported)
plots["export"].set_value(export_emissions_series, year,
exported.sum() / base_E_sum)
plots["export"].set_value(export_dollar_series, year,
y.get_exports().sum() / base_gdp)
# prepare other country values
stmt = db.prepare("""SELECT from_sector, sum(value)
FROM wiod_plus.%s_io_import_%d WHERE country = $1
AND from_sector NOT IN ('ITM', 'IMP', 'Rex')
GROUP BY from_sector""" % (base_country.lower(), year))
imported_E = None
imported_y = None
domestic_E = None # emissions under domestic technology assumption
for country in other_countries:
envgen.set_condition_args(country)
envgen.set_sectors(common.default_env_sectors)
iogen.set_condition_args(country)
sectors = iogen.get_sectors(
) # number of sectors varies by country
E = envgen.get_env_vector(env_series)
if E.mat() is None: # this country has no data for env series
continue
imports = NamedMatrix(rows=sectors, cols=["imports"])
db_result = stmt(country)
if not len(db_result):
# we can't do any of the following with a blank import vector
continue
for row in db_result:
imports.set_element(row[0], "imports", row[1])
sel = common.get_io_harmonizer(iogen)
x = iogen.get_x()
L = iogen.get_L()
J = E.divide(sel.matrix_mult(x), ignore_zero_denom=True)
JsL = J.square_matrix_from_diag().matrix_mult(sel).matrix_mult(L)
current_E = JsL.matrix_mult(imports)
current_y = sel.matrix_mult(imports)
# temporary dumb way to deal with sector mismatch
compat_imports = NamedMatrix(rows=base_JsL.get_columns(),
cols=["imports"])
for col in base_JsL.get_columns():
if col in sectors:
compat_imports.set_element(col, "imports",
imports.get_element(col))
current_domestic_E = base_JsL.matrix_mult(compat_imports)
imported_E = sum_if_not_none(imported_E, current_E)
imported_y = sum_if_not_none(imported_y, current_y)
domestic_E = sum_if_not_none(domestic_E, current_domestic_E)
exported_y.set_element(country, str(year), imports.sum())
exported_E.set_element(country, str(year), current_E.sum())
# populate results table
TradeResultsTable.insert_exports(year, base_country, exported)
TradeResultsTable.insert_imports(year, base_country, imported_E)
# generate import plots
for sector in common.default_env_sectors:
base_y_val = base_y.get_element(sector)
if base_y_val > 0:
plots[sector].set_value(
import_dollar_series, year,
imported_y.get_element(sector) / base_y_val)
base_E_val = base_E.get_element(sector)
if base_E_val > 0:
plots[sector].set_value(
emission_series, year,
imported_E.get_element(sector) / base_E_val)
plots[sector].set_value(
self_emission_series, year,
domestic_E.get_element(sector) / base_E_val)
plots["import"].set_value(import_dollar_series, year,
imported_y.sum() / base_y.sum())
plots["import"].set_value(emission_series, year,
imported_E.sum() / base_E_sum)
plots["import"].set_value(self_emission_series, year,
domestic_E.sum() / base_E_sum)
if year in dta_plots and base_country in dta_countries:
dta_plots[year].set_value("DTA", config.countries[base_country],
imported_E.sum() / base_E_sum)
dta_plots[year].set_value("No DTA", config.countries[base_country],
domestic_E.sum() / base_E_sum)
# this is for DTA vs non-DTA table
#print(base_country, year,
# imported_E.sum(),
# domestic_E.sum(),
# imported_E.sum() / base_E_sum,
# domestic_E.sum() / base_E_sum)
plots["export"].set_value(emission_series, year,
imported_E.sum() / base_E_sum)
import_balance.set_column(str(year), imported_E)
#describe_exporters()
#create_plots()
#describe_balance(True)
describe_balance_intensity()
#describe_exporters()
#create_plots()
#describe_balance(True)
describe_balance_intensity()
# main
base_countries = config.countries.keys()
#env_keys = config.env_series_names.keys():
env_keys = [
"CO2",
]
for year in [1995, 2009]:
dta_plots[year] = GNUPlot("dta vs fta %d" % year, None, "wiod")
dta_countries = [
"AUS", "AUT", "BEL", "BRA", "CAN", "DEU", "DNK", "ESP", "FIN", "FRA",
"GBR", "GRC", "IDN", "IND", "ITA", "JPN", "KOR", "MEX", "NLD", "POL",
"ROU", "RUS", "CHN", "TUR", "TWN", "USA"
]
#for year in [1995, 2009]:
# balance_plots[year] = ScatterPlot(
# "balance vs intensity %d" % year, None, "wiod")
#TradeResultsTable.activate()
for base_country in base_countries:
if base_country in config.bad_data_blacklist:
def graph_table(filename, title, vector, base_year, intensity_divisor,
sector_groups):
plot = GNUPlot(filename, title, "usa")
data = {}
sortby = {}
numerators = {}
denominators = {}
for key in sector_groups.keys():
data[key] = {}
denominators[key] = dict((year, 0) for year in config.STUDY_YEARS)
numerators[key] = dict((year, 0) for year in config.STUDY_YEARS)
max_year = max(config.STUDY_YEARS)
for sector in vector.keys():
print(sector)
numerator = vector[sector]
for (key, sectors) in sector_groups.items():
if sector in sectors:
for year in config.STUDY_YEARS:
if intensity_divisor is not None:
denominators[key][year] += \
intensity_divisor[sector][year]
else:
denominators[key][year] += 1
numerators[key][year] += numerator[year]
break
for key in sector_groups.keys():
print(key, denominators[key])
data[key] = dict(
(year, numerators[key][year] / denominators[key][year])
for year in config.STUDY_YEARS)
sortby[data[key][max_year] / data[key][base_year]] = key
sorted_groups = [
sortby[key] for key in sorted(sortby.keys(), reverse=True)
]
plot.add_custom_setup("set style data linespoints")
plot.add_custom_setup("unset colorbox")
plot.add_custom_setup("set grid")
min_x = 5 * math.floor(min(config.STUDY_YEARS) / 5)
max_x = 5 * math.ceil(max(config.STUDY_YEARS) / 5) + 5 # room for labels
plot.add_custom_setup("set xrange [ %d : %d ]" % (min_x, max_x))
interval = 1 / (len(sorted_groups) - 1)
for i in range(len(sorted_groups)):
plot.add_custom_setup("set style lines %d lc palette frac %.2f lw 2" %
(i + 1, i * interval))
series_values = []
def name_for_group(group):
if group in bea.short_nipa:
group_name = bea.short_nipa[group]
else:
group_name = group
return group_name
for i in range(len(sorted_groups)):
group = sorted_groups[i]
group_name = name_for_group(group)
series_values.append(group_name)
plot.suppress_title(group_name)
base_value = data[group][base_year]
for year in config.STUDY_YEARS:
plot.set_value(group_name, year,
data[group][year] / base_value * 100)
plot.set_series_style(group_name, "linestyle %d" % (i + 1))
plot.series_values = series_values
plot.write_tables()
plot.get_axis_specs() # make sure max_y is adjusted
prev_pos = None
for i in range(len(sorted_groups)):
group = sorted_groups[i]
group_name = name_for_group(group)
ending_value = data[group][max_year] / data[group][base_year] * 100
position = ending_value / plot.max_y + 0.01 # line up baseline
# space labels out by at least 0.04 so they don't overlap
if prev_pos is not None and prev_pos - position < 0.03:
position = prev_pos - 0.03
plot.add_custom_setup(
"set label %d '%s' at graph 0.81, %.2f font 'Arial,8'" %
(i + 1, group_name, position))
prev_pos = position
plot.generate_plot()
"style": 'pointtype 7 linetype rgb "red"', # solid circle
},
}
for year in config.STUDY_YEARS:
# show some output since this script takes awhile
print(year)
iogen = common.iogen_for_year(year)
envgen = common.envgen_for_year(year)
env_sectors = common.env_sectors_for_year(year)
for (country, country_name) in config.countries.items():
if country not in plots:
plots[country] = GNUPlot("%s_%s" % (env_series, country),
"%s in %s" % (env_series, country_name),
"%s-totals" % env_series)
plot = plots[country]
for (series, specs) in series_specs.items():
plot.set_series_style(specs["title"], specs["style"])
marginal_plots[country] = GNUPlot(
"%s_%s" % (env_series, country),
"%s in %s" % (env_series, country_name), env_series)
mplot = marginal_plots[country]
for (series, specs) in mseries_specs.items():
mplot.set_series_style(specs["title"], specs["style"])
else:
mplot = marginal_plots[country]
plot = plots[country]
def do_plots():
for (name, measurements) in config.env_series_names.items():
data = {}
for year in config.STUDY_YEARS:
strings = {
"schema": config.WIOD_SCHEMA,
"year": year,
"fd_sectors": sqlhelper.set_repr(config.default_fd_sectors),
"measurements": sqlhelper.set_repr(measurements),
"nipa_schema": usa.config.NIPA_SCHEMA,
}
stmt = db.prepare(
"""SELECT a.country, a.series, b.gdp,
a.series / b.gdp as intensity
FROM (SELECT country, sum(value) as series
FROM %(schema)s.env_%(year)d
WHERE industry = 'total'
AND measurement in %(measurements)s
GROUP BY country) a,
(SELECT aa.country, sum(value) * deflator as gdp
FROM %(schema)s.indbyind_%(year)d aa,
(SELECT 100 / gdp as deflator
FROM %(nipa_schema)s.implicit_price_deflators
WHERE year = $1) bb
WHERE to_ind in %(fd_sectors)s
GROUP BY aa.country, deflator) b
WHERE a.country = b.country
AND a.series is not null
ORDER BY a.series / b.gdp""" % strings)
for row in stmt(year):
country = row[0]
intensity = row[3]
if country not in data:
data[country] = {}
data[country][year] = intensity
slopes = {}
for (country, country_data) in data.items():
n = len(country_data.keys())
if n < 2:
continue
sum_y = sum(country_data.values())
sum_x = sum(country_data.keys())
slope = (n * sum([k * v for (k, v) in country_data.items()]) \
- sum_x * sum_y) / \
(n * sum([k * k for k in country_data.keys()]) - sum_x)
slopes[country] = slope * 1000000
years = "%d-%d" % (config.STUDY_YEARS[0], config.STUDY_YEARS[-1])
i = 0
binsize = 8
plot = None
for (country, slope) in sorted(slopes.items(), key=lambda x: x[1]):
if i % binsize == 0:
if plot is not None:
plot.write_tables()
plot.generate_plot()
tier = i / binsize + 1
plot = GNUPlot("tier%d" % tier, "",
#"%s intensity from %s, tier %d" \
# % (name, years, tier),
"wiod-%s" % name.replace(" ", "-"))
plot.legend("width -5")
for year in config.STUDY_YEARS:
if year in data[country]:
plot.set_value(
"%s (%.2f)" % (config.countries[country], slope),
year,
data[country][year])
i += 1
if plot is not None:
plot.write_tables()
plot.generate_plot()
