def graph_table(filename, title, vector, base_year, is_intensities, collapse=[]): plot = GNUPlot(filename, title, "usa") data = {} sortby = {} other_category = "Other" max_year = max(config.STUDY_YEARS) other_category_numer = dict((year, 0) for year in config.STUDY_YEARS) other_category_denom = dict((year, 0) for year in config.STUDY_YEARS) for group in vector.keys(): numerator = vector[group] if is_intensities: denominator = pce_dollars[group] else: denominator = dict((year, 1) for year in config.STUDY_YEARS) base_value = numerator[base_year] / denominator[base_year] if group in collapse: for year in config.STUDY_YEARS: other_category_numer[year] += numerator[year] # adding 1 for each year is fine since it's relative to the base other_category_denom[year] += denominator[year] else: data[group] = {} for year in config.STUDY_YEARS: data[group][year] = numerator[year] / denominator[year] sortby[ data[group][max_year] / data[group][base_year] ] = group if len(other_category_numer): data[other_category] = dict( (year, other_category_numer[year] / other_category_denom[year]) \ for year in config.STUDY_YEARS) sortby[ data[other_category][max_year] /\ data[other_category][base_year] ] = other_category sorted_groups = [sortby[key] for key in sorted(sortby.keys(), reverse=True)] plot.add_custom_setup("set style data linespoints") plot.add_custom_setup("unset colorbox") plot.add_custom_setup("set grid") min_x = 5 * math.floor(min(config.STUDY_YEARS) / 5) max_x = 5 * math.ceil(max(config.STUDY_YEARS) / 5) + 5 # room for labels plot.add_custom_setup("set xrange [ %d : %d ]" % (min_x, max_x)) interval = 1 / (len(sorted_groups) - 1) for i in range(len(sorted_groups)): plot.add_custom_setup("set style lines %d lc palette frac %.2f lw 2" % (i + 1, i * interval)) series_values = [] for i in range(len(sorted_groups)): group = sorted_groups[i] if group in bea.short_nipa: group_name = bea.short_nipa[group] else: group_name = group series_values.append(group_name) plot.suppress_title(group_name) base_value = data[group][base_year] for year in config.STUDY_YEARS: print(group, year, data[group][year], base_value, data[group][year] / base_value) plot.set_value(group_name, year, data[group][year] / base_value * 100) plot.set_series_style(group_name, "linestyle %d" % (i + 1)) plot.series_values = series_values plot.write_tables() plot.get_axis_specs() # make sure max_y is adjusted prev_pos = None for i in range(len(sorted_groups)): group = sorted_groups[i] if group in bea.short_nipa: group_name = bea.short_nipa[group] else: group_name = group ending_value = data[group][max_year] / data[group][base_year] * 100 position = ending_value / plot.max_y + 0.01 # line up baseline # space labels out by at least 0.04 so they don't overlap if prev_pos is not None and prev_pos - position < 0.03: position = prev_pos - 0.03 plot.add_custom_setup( "set label %d '%s' at graph 0.81, %.2f font 'Arial,8'" % (i + 1, group_name, position)) prev_pos = position plot.generate_plot()
content = GNUPlot.get_plotscript_content(self) self.writing_plotscript = False return content def get_plot_clauses(self): clauses = GNUPlot.get_plot_clauses(self) column = len(self.xvalues) + 1 clauses.append( "'' using %d axis x1y2 with linespoints lc rgb 'black' title column(%d)" % (column, column)) return clauses dirname = "usa-pce-energy" ff_plot = GNUPlot("total-ff", "Fossil fuels (trillion Btu)", dirname) elec_plot = GNUPlot("total-elec", "Electricity (trillion Btu)", dirname) ff_iplot = GNUPlot("total-intensity-ff", "Fossil fuels (Btu per dollar)", dirname) elec_iplot = GNUPlot("total-intensity-elec", "Electricity (Btu per dollar)", dirname) ff_plot.style = "histogram" ff_iplot.style = "histogram" elec_plot.style = "histogram rowstacked" elec_iplot.style = "histogram rowstacked" for year in config.STUDY_YEARS: # args: year, is_hybrid, allow_imports, adjust for inflation iogen = common.iogen_for_year(year, True, True, True)
fd_dollars = { "exports": export_n.sum(), "pce": Yn.get_pce().sum(), "imports": import_n.sum(), } ###### breakdown by fd sector dirname = "usa-pce-energy" for (key, vector) in fd_vectors.items(): fdgroup = bea.fd_sector_names[key] ikey = key + "-intensity" if key not in plots: plots[key] = GNUPlot(key, fdgroup + " (trillion Btu)", dirname) plots[key].style = "histogram" if ikey not in plots: plots[ikey] = GNUPlot(ikey, fdgroup + " (MMBtu per 2005$)", dirname) plots[ikey].style = "histogram" use_vector = L.matrix_mult(vector) for i in range(len(energy_codes)): code = energy_codes[i] name = eia.name_for_naics(code) value = use_vector.get_element(rowname=code) # bBtu # divide bBtu by 1000 for tBtu plots[key].set_value(year, name, value / 1000) # div bBtu by k$ for MMBtu/$
def trade_ratios(base_country, env_key): env_series = config.env_series_names[env_key] other_countries = sorted(config.countries.keys()) other_countries.remove(base_country) yearstrings = [str(y) for y in config.STUDY_YEARS] exported_y = NamedMatrix(rows=other_countries, cols=yearstrings) exported_E = NamedMatrix(rows=other_countries, cols=yearstrings) def describe_exporters(): print("Exports by country - " + env_key) print("dollars") for country in other_countries: formatted = [ ("%.0f" % exported_y.get_element(country, year)).rjust(6) for year in yearstrings ] print(country, " ".join(formatted)) print("emissions") for country in other_countries: formatted = [ ("%.0f" % exported_E.get_element(country, year)).rjust(6) for year in yearstrings ] print(country, " ".join(formatted)) print("intensities") intensities = exported_E.divide(exported_y) for country in other_countries: formatted = [ ("%.2f" % intensities.get_element(country, year)).rjust(6) for year in yearstrings ] print(country, " ".join(formatted)) ### prepare plots env_title = env_key.replace(" ", "-") plot_group = "import-%s-%s" % (env_title, base_country) plots = {} for sector in common.default_env_sectors: sector_title = common.get_industry_title(sector) plots[sector] = GNUPlot( "%s_%s" % (env_title, sector), "", #"Imports of %s - %s" % (sector_title, env_key), plot_group) plots["import"] = GNUPlot( "%s_import" % env_title, "", #"All imports - %s" % env_key, plot_group) plots["export"] = GNUPlot( "%s_export" % env_title, "", #"All imports - %s" % env_key, plot_group) def create_plots(): for (sector, plot) in plots.items(): plot.set_series_style( import_dollar_series, "pointtype 5 linetype rgb '#88BBFF'") # hollow circle plot.set_series_style( emission_series, "pointtype 7 linetype rgb '#1144FF'") # solid circle plot.set_series_style(self_emission_series, "pointtype 12 linetype rgb '#0000FF'") plot.set_series_style(export_dollar_series, "pointtype 4 linetype rgb '#CC9933'") plot.set_series_style(export_emissions_series, "pointtype 7 linetype rgb '#996600'") #plot.legend("width -8") plot.width = 480 plot.height = 300 plot.write_tables() plot.generate_plot() ### trade balance matrices export_balance = NamedMatrix(rows=common.default_env_sectors, cols=yearstrings) import_balance = NamedMatrix(rows=common.default_env_sectors, cols=yearstrings) all_E = NamedMatrix(rows=common.default_env_sectors, cols=yearstrings) def describe_balance(ratios=False): export_total = export_balance.sum(0) import_total = import_balance.sum(0) all_total = all_E.sum(0) balance = export_total.subtract(import_total) country_name = config.countries[base_country] oddyears = [ str(y) for y in filter(lambda x: x % 2 == 1, config.STUDY_YEARS) ] if ratios: balance_ratio = balance.divide(all_total) print( country_name.ljust(15) + " & " + " & ".join([("%.2f" % balance_ratio.get_element("sum", y)).rjust(6) for y in oddyears]) + " \\\\") else: print( country_name.ljust(15) + " & " + " & ".join([ utils.add_commas(balance.get_element("sum", y)).rjust(9) for y in oddyears ]) + " \\\\") def describe_balance_intensity(): export_total = export_balance.sum(0) import_total = import_balance.sum(0) all_total = all_E.sum(0) balance = export_total.subtract(import_total) balance_ratio = balance.divide(all_total) country_name = config.countries[base_country] years = [str(minyear), str(maxyear)] fields = [] for y in years: balance_val = balance.get_element("sum", y) ratio_val = balance_ratio.get_element("sum", y) (gdp_val, env_val, intensity) = \ common.get_efficiency(base_country, int(y)) if int(y) in balance_plots: plot = balance_plots[int(y)] # ratio = exports to imports plot.set_value("2 ratio", base_country, ratio_val) plot.set_value("1 intensity", base_country, intensity) if int(y) in worldmap: worldmap[int(y)].set_country_value(base_country, balance_val) fields.append(utils.add_commas(balance_val)) fields.append("%.2f" % ratio_val) fields.append("%.2f" % intensity) print(country_name.ljust(15) + " & " + " & ".join(fields) + " \\NN") ### iogen = common.iogen_for_year(config.STUDY_YEARS[0]) envgen = common.envgen_for_year(config.STUDY_YEARS[0]) strings = {"schema": config.WIOD_SCHEMA} for year in config.STUDY_YEARS: strings["year"] = year base_E = get_wiod_env_vector(base_country, year, env_series) all_E.set_column(str(year), base_E) base_E_sum = base_E.sum() iogen.set_table("%(schema)s.indbyind_%(year)d" % strings) iogen.set_condition_args(base_country) envgen.set_table("%(schema)s.env_%(year)d" % strings) envgen.set_condition_args(base_country) envgen.set_sectors(common.default_env_sectors) # prepare base country values y = iogen.get_Y() import_column = common.get_import_vector(iogen) base_imports = import_column.sum() base_gdp = y.sum() harmonizer = common.get_io_harmonizer(iogen) base_y = harmonizer.matrix_mult(y.get_total()) x = iogen.get_x() E = envgen.get_env_vector(env_series) L = iogen.get_L() J = E.divide(harmonizer.matrix_mult(x), ignore_zero_denom=True) base_JsL = J.square_matrix_from_diag()\ .matrix_mult(harmonizer).matrix_mult(L) exported = base_JsL.matrix_mult(y.get_exports()) export_balance.set_column(str(year), exported) plots["export"].set_value(export_emissions_series, year, exported.sum() / base_E_sum) plots["export"].set_value(export_dollar_series, year, y.get_exports().sum() / base_gdp) # prepare other country values stmt = db.prepare("""SELECT from_sector, sum(value) FROM wiod_plus.%s_io_import_%d WHERE country = $1 AND from_sector NOT IN ('ITM', 'IMP', 'Rex') GROUP BY from_sector""" % (base_country.lower(), year)) imported_E = None imported_y = None domestic_E = None # emissions under domestic technology assumption for country in other_countries: envgen.set_condition_args(country) envgen.set_sectors(common.default_env_sectors) iogen.set_condition_args(country) sectors = iogen.get_sectors( ) # number of sectors varies by country E = envgen.get_env_vector(env_series) if E.mat() is None: # this country has no data for env series continue imports = NamedMatrix(rows=sectors, cols=["imports"]) db_result = stmt(country) if not len(db_result): # we can't do any of the following with a blank import vector continue for row in db_result: imports.set_element(row[0], "imports", row[1]) sel = common.get_io_harmonizer(iogen) x = iogen.get_x() L = iogen.get_L() J = E.divide(sel.matrix_mult(x), ignore_zero_denom=True) JsL = J.square_matrix_from_diag().matrix_mult(sel).matrix_mult(L) current_E = JsL.matrix_mult(imports) current_y = sel.matrix_mult(imports) # temporary dumb way to deal with sector mismatch compat_imports = NamedMatrix(rows=base_JsL.get_columns(), cols=["imports"]) for col in base_JsL.get_columns(): if col in sectors: compat_imports.set_element(col, "imports", imports.get_element(col)) current_domestic_E = base_JsL.matrix_mult(compat_imports) imported_E = sum_if_not_none(imported_E, current_E) imported_y = sum_if_not_none(imported_y, current_y) domestic_E = sum_if_not_none(domestic_E, current_domestic_E) exported_y.set_element(country, str(year), imports.sum()) exported_E.set_element(country, str(year), current_E.sum()) # populate results table TradeResultsTable.insert_exports(year, base_country, exported) TradeResultsTable.insert_imports(year, base_country, imported_E) # generate import plots for sector in common.default_env_sectors: base_y_val = base_y.get_element(sector) if base_y_val > 0: plots[sector].set_value( import_dollar_series, year, imported_y.get_element(sector) / base_y_val) base_E_val = base_E.get_element(sector) if base_E_val > 0: plots[sector].set_value( emission_series, year, imported_E.get_element(sector) / base_E_val) plots[sector].set_value( self_emission_series, year, domestic_E.get_element(sector) / base_E_val) plots["import"].set_value(import_dollar_series, year, imported_y.sum() / base_y.sum()) plots["import"].set_value(emission_series, year, imported_E.sum() / base_E_sum) plots["import"].set_value(self_emission_series, year, domestic_E.sum() / base_E_sum) if year in dta_plots and base_country in dta_countries: dta_plots[year].set_value("DTA", config.countries[base_country], imported_E.sum() / base_E_sum) dta_plots[year].set_value("No DTA", config.countries[base_country], domestic_E.sum() / base_E_sum) # this is for DTA vs non-DTA table #print(base_country, year, # imported_E.sum(), # domestic_E.sum(), # imported_E.sum() / base_E_sum, # domestic_E.sum() / base_E_sum) plots["export"].set_value(emission_series, year, imported_E.sum() / base_E_sum) import_balance.set_column(str(year), imported_E) #describe_exporters() #create_plots() #describe_balance(True) describe_balance_intensity()
#describe_exporters() #create_plots() #describe_balance(True) describe_balance_intensity() # main base_countries = config.countries.keys() #env_keys = config.env_series_names.keys(): env_keys = [ "CO2", ] for year in [1995, 2009]: dta_plots[year] = GNUPlot("dta vs fta %d" % year, None, "wiod") dta_countries = [ "AUS", "AUT", "BEL", "BRA", "CAN", "DEU", "DNK", "ESP", "FIN", "FRA", "GBR", "GRC", "IDN", "IND", "ITA", "JPN", "KOR", "MEX", "NLD", "POL", "ROU", "RUS", "CHN", "TUR", "TWN", "USA" ] #for year in [1995, 2009]: # balance_plots[year] = ScatterPlot( # "balance vs intensity %d" % year, None, "wiod") #TradeResultsTable.activate() for base_country in base_countries: if base_country in config.bad_data_blacklist:
def graph_table(filename, title, vector, base_year, intensity_divisor, sector_groups): plot = GNUPlot(filename, title, "usa") data = {} sortby = {} numerators = {} denominators = {} for key in sector_groups.keys(): data[key] = {} denominators[key] = dict((year, 0) for year in config.STUDY_YEARS) numerators[key] = dict((year, 0) for year in config.STUDY_YEARS) max_year = max(config.STUDY_YEARS) for sector in vector.keys(): print(sector) numerator = vector[sector] for (key, sectors) in sector_groups.items(): if sector in sectors: for year in config.STUDY_YEARS: if intensity_divisor is not None: denominators[key][year] += \ intensity_divisor[sector][year] else: denominators[key][year] += 1 numerators[key][year] += numerator[year] break for key in sector_groups.keys(): print(key, denominators[key]) data[key] = dict( (year, numerators[key][year] / denominators[key][year]) for year in config.STUDY_YEARS) sortby[data[key][max_year] / data[key][base_year]] = key sorted_groups = [ sortby[key] for key in sorted(sortby.keys(), reverse=True) ] plot.add_custom_setup("set style data linespoints") plot.add_custom_setup("unset colorbox") plot.add_custom_setup("set grid") min_x = 5 * math.floor(min(config.STUDY_YEARS) / 5) max_x = 5 * math.ceil(max(config.STUDY_YEARS) / 5) + 5 # room for labels plot.add_custom_setup("set xrange [ %d : %d ]" % (min_x, max_x)) interval = 1 / (len(sorted_groups) - 1) for i in range(len(sorted_groups)): plot.add_custom_setup("set style lines %d lc palette frac %.2f lw 2" % (i + 1, i * interval)) series_values = [] def name_for_group(group): if group in bea.short_nipa: group_name = bea.short_nipa[group] else: group_name = group return group_name for i in range(len(sorted_groups)): group = sorted_groups[i] group_name = name_for_group(group) series_values.append(group_name) plot.suppress_title(group_name) base_value = data[group][base_year] for year in config.STUDY_YEARS: plot.set_value(group_name, year, data[group][year] / base_value * 100) plot.set_series_style(group_name, "linestyle %d" % (i + 1)) plot.series_values = series_values plot.write_tables() plot.get_axis_specs() # make sure max_y is adjusted prev_pos = None for i in range(len(sorted_groups)): group = sorted_groups[i] group_name = name_for_group(group) ending_value = data[group][max_year] / data[group][base_year] * 100 position = ending_value / plot.max_y + 0.01 # line up baseline # space labels out by at least 0.04 so they don't overlap if prev_pos is not None and prev_pos - position < 0.03: position = prev_pos - 0.03 plot.add_custom_setup( "set label %d '%s' at graph 0.81, %.2f font 'Arial,8'" % (i + 1, group_name, position)) prev_pos = position plot.generate_plot()
"style": 'pointtype 7 linetype rgb "red"', # solid circle }, } for year in config.STUDY_YEARS: # show some output since this script takes awhile print(year) iogen = common.iogen_for_year(year) envgen = common.envgen_for_year(year) env_sectors = common.env_sectors_for_year(year) for (country, country_name) in config.countries.items(): if country not in plots: plots[country] = GNUPlot("%s_%s" % (env_series, country), "%s in %s" % (env_series, country_name), "%s-totals" % env_series) plot = plots[country] for (series, specs) in series_specs.items(): plot.set_series_style(specs["title"], specs["style"]) marginal_plots[country] = GNUPlot( "%s_%s" % (env_series, country), "%s in %s" % (env_series, country_name), env_series) mplot = marginal_plots[country] for (series, specs) in mseries_specs.items(): mplot.set_series_style(specs["title"], specs["style"]) else: mplot = marginal_plots[country] plot = plots[country]
def do_plots(): for (name, measurements) in config.env_series_names.items(): data = {} for year in config.STUDY_YEARS: strings = { "schema": config.WIOD_SCHEMA, "year": year, "fd_sectors": sqlhelper.set_repr(config.default_fd_sectors), "measurements": sqlhelper.set_repr(measurements), "nipa_schema": usa.config.NIPA_SCHEMA, } stmt = db.prepare( """SELECT a.country, a.series, b.gdp, a.series / b.gdp as intensity FROM (SELECT country, sum(value) as series FROM %(schema)s.env_%(year)d WHERE industry = 'total' AND measurement in %(measurements)s GROUP BY country) a, (SELECT aa.country, sum(value) * deflator as gdp FROM %(schema)s.indbyind_%(year)d aa, (SELECT 100 / gdp as deflator FROM %(nipa_schema)s.implicit_price_deflators WHERE year = $1) bb WHERE to_ind in %(fd_sectors)s GROUP BY aa.country, deflator) b WHERE a.country = b.country AND a.series is not null ORDER BY a.series / b.gdp""" % strings) for row in stmt(year): country = row[0] intensity = row[3] if country not in data: data[country] = {} data[country][year] = intensity slopes = {} for (country, country_data) in data.items(): n = len(country_data.keys()) if n < 2: continue sum_y = sum(country_data.values()) sum_x = sum(country_data.keys()) slope = (n * sum([k * v for (k, v) in country_data.items()]) \ - sum_x * sum_y) / \ (n * sum([k * k for k in country_data.keys()]) - sum_x) slopes[country] = slope * 1000000 years = "%d-%d" % (config.STUDY_YEARS[0], config.STUDY_YEARS[-1]) i = 0 binsize = 8 plot = None for (country, slope) in sorted(slopes.items(), key=lambda x: x[1]): if i % binsize == 0: if plot is not None: plot.write_tables() plot.generate_plot() tier = i / binsize + 1 plot = GNUPlot("tier%d" % tier, "", #"%s intensity from %s, tier %d" \ # % (name, years, tier), "wiod-%s" % name.replace(" ", "-")) plot.legend("width -5") for year in config.STUDY_YEARS: if year in data[country]: plot.set_value( "%s (%.2f)" % (config.countries[country], slope), year, data[country][year]) i += 1 if plot is not None: plot.write_tables() plot.generate_plot()