def _agg_hist(self):
# aggregate and clean hist
_log('Aggregating historical values to native regions')
# must set verify to false for now because some isos aren't included!
self.hist = utils.agg_regions(
self.hist, verify=False, mapping=self.regions,
rfrom='ISO Code', rto='Native Region Code'
)
def test_region_agg_funky_name():
df = pd.DataFrame({
'sector': ['foo', 'foo'],
'region': ['a', 'b'],
'2010': [1.0, 4.0],
'units': ['Mt'] * 2,
'gas': ['BC'] * 2,
}).set_index(utils.df_idx).sort_index()
mapping = pd.DataFrame([['fOO_Bar', 'a'], ['fOO_Bar', 'b']],
columns=['x', 'y'])
exp = pd.DataFrame({
'sector': ['foo'],
'region': ['fOO_Bar'],
'2010': [5.0],
'units': ['Mt'],
'gas': ['BC'],
}).set_index(utils.df_idx).sort_index()
obs = utils.agg_regions(df, rfrom='y', rto='x', mapping=mapping)
pdt.assert_frame_equal(obs, exp)
def _harmonize_regions(config, prefix, suffix, regions, hist, model, overrides,
base_year, add_5regions):
# clean model
model = utils.subtract_regions_from_world(model, 'model', base_year)
model = utils.remove_recalculated_sectors(model, prefix, suffix)
# remove rows with all 0s
model = model[(model.T > 0).any()]
# clean hist
hist = utils.subtract_regions_from_world(hist, 'hist', base_year)
hist = utils.remove_recalculated_sectors(hist, prefix, suffix)
# remove rows with all 0s
hist = hist[(hist.T > 0).any()]
if model.empty:
raise RuntimeError(
'Model is empty after downselecting regional values')
# harmonize
utils.check_null(model, 'model')
utils.check_null(hist, 'hist', fail=True)
harmonizer = Harmonizer(model, hist, config=config)
_log('Harmonizing (with example methods):')
_log(harmonizer.methods(overrides=overrides).head())
if overrides is not None:
_log('and override methods:')
_log(overrides.head())
model = harmonizer.harmonize(overrides=overrides)
utils.check_null(model, 'model')
metadata = harmonizer.metadata()
# add aggregate variables
totals = '|'.join([prefix, suffix])
if model.index.get_level_values('sector').isin([totals]).any():
msg = 'Removing sector aggregates. Recalculating with harmonized totals.'
_warn(msg)
model.drop(totals, level='sector', inplace=True)
model = (
utils.EmissionsAggregator(model)
.add_variables(totals=totals, aggregates=False)
.df
.set_index(utils.df_idx)
)
utils.check_null(model, 'model')
# combine regional values to send back into template form
model.reset_index(inplace=True)
model = model.set_index(utils.df_idx).sort_index()
glb = utils.combine_rows(model, 'region', 'World',
sumall=True, rowsonly=True)
model = glb.combine_first(model)
# add 5regions
if add_5regions:
_log('Adding 5region values')
# explicitly don't add World, it already exists from aggregation
mapping = regions[regions['Native Region Code'] != 'World'].copy()
aggdf = utils.agg_regions(model, mapping=mapping,
rfrom='Native Region Code', rto='5_region')
model = model.append(aggdf)
assert(not model.isnull().values.any())
# duplicates come in from World and World being translated
duplicates = model.index.duplicated(keep='first')
if duplicates.any():
regions = model[duplicates].index.get_level_values('region').unique()
msg = 'Dropping duplicate rows found for regions: {}'.format(regions)
_warn(msg)
model = model[~duplicates]
return model, metadata