def _harmonize_global_total(config, prefix, suffix, hist, model, overrides):
gases = utils.harmonize_total_gases
sector = '|'.join([prefix, suffix])
idx = (pd.IndexSlice['World', gases, sector],
pd.IndexSlice[:])
h = hist.loc[idx].copy()
try:
m = model.loc[idx].copy()
except TypeError:
_warn('Non-history gases not found in model')
return None, None
if m.empty:
return None, None
# catch empty dfs if no global toatls are overriden
if overrides is None:
o = None
else:
gases = overrides.index.get_level_values('gas').intersection(gases)
try:
gases = gases if len(gases) > 1 else gases[0]
except IndexError: # thrown if no harmonize_total_gases
o = None
idx = (pd.IndexSlice['World', gases, sector],
pd.IndexSlice[:])
try:
o = overrides.loc[idx].copy()
except TypeError: # thrown if gases not
o = None
utils.check_null(m, 'model')
utils.check_null(h, 'hist', fail=True)
harmonizer = Harmonizer(m, h, config=config)
_log('Harmonizing (with example methods):')
_log(harmonizer.methods(overrides=o).head())
if o is not None:
_log('and override methods:')
_log(o.head())
m = harmonizer.harmonize(overrides=o)
utils.check_null(m, 'model')
metadata = harmonizer.metadata()
return m, metadata
def _harmonize_global_total(config,
prefix,
suffix,
hist,
model,
overrides,
default_global_gases=True):
all_gases = list(model.index.get_level_values('gas').unique())
gases = utils.harmonize_total_gases if default_global_gases else all_gases
sector = '|'.join([prefix, suffix])
idx = (pd.IndexSlice['World', gases, sector], pd.IndexSlice[:])
h = hist.loc[idx].copy()
try:
m = model.loc[idx].copy()
except TypeError:
_warn('Non-history gases not found in model')
return None, None
if m.empty:
return None, None
# match override methods with global gases, None if no match
o = _get_global_overrides(overrides, gases, sector)
utils.check_null(m, 'model')
utils.check_null(h, 'hist', fail=True)
harmonizer = Harmonizer(m, h, config=config)
_log('Harmonizing (with example methods):')
_log(harmonizer.methods(overrides=o).head())
if o is not None:
_log('and override methods:')
_log(o.head())
m = harmonizer.harmonize(overrides=o)
utils.check_null(m, 'model')
metadata = harmonizer.metadata()
return m, metadata
def _harmonize_regions(config, prefix, suffix, regions, hist, model, overrides,
base_year, add_5regions):
# clean model
model = utils.subtract_regions_from_world(model, 'model', base_year)
model = utils.remove_recalculated_sectors(model, prefix, suffix)
# remove rows with all 0s
model = model[(model.T > 0).any()]
# clean hist
hist = utils.subtract_regions_from_world(hist, 'hist', base_year)
hist = utils.remove_recalculated_sectors(hist, prefix, suffix)
# remove rows with all 0s
hist = hist[(hist.T > 0).any()]
if model.empty:
raise RuntimeError(
'Model is empty after downselecting regional values')
# harmonize
utils.check_null(model, 'model')
utils.check_null(hist, 'hist', fail=True)
harmonizer = Harmonizer(model, hist, config=config)
_log('Harmonizing (with example methods):')
_log(harmonizer.methods(overrides=overrides).head())
if overrides is not None:
_log('and override methods:')
_log(overrides.head())
model = harmonizer.harmonize(overrides=overrides)
utils.check_null(model, 'model')
metadata = harmonizer.metadata()
# add aggregate variables
totals = '|'.join([prefix, suffix])
if model.index.get_level_values('sector').isin([totals]).any():
msg = 'Removing sector aggregates. Recalculating with harmonized totals.'
_warn(msg)
model.drop(totals, level='sector', inplace=True)
model = (
utils.EmissionsAggregator(model)
.add_variables(totals=totals, aggregates=False)
.df
.set_index(utils.df_idx)
)
utils.check_null(model, 'model')
# combine regional values to send back into template form
model.reset_index(inplace=True)
model = model.set_index(utils.df_idx).sort_index()
glb = utils.combine_rows(model, 'region', 'World',
sumall=True, rowsonly=True)
model = glb.combine_first(model)
# add 5regions
if add_5regions:
_log('Adding 5region values')
# explicitly don't add World, it already exists from aggregation
mapping = regions[regions['Native Region Code'] != 'World'].copy()
aggdf = utils.agg_regions(model, mapping=mapping,
rfrom='Native Region Code', rto='5_region')
model = model.append(aggdf)
assert(not model.isnull().values.any())
# duplicates come in from World and World being translated
duplicates = model.index.duplicated(keep='first')
if duplicates.any():
regions = model[duplicates].index.get_level_values('region').unique()
msg = 'Dropping duplicate rows found for regions: {}'.format(regions)
_warn(msg)
model = model[~duplicates]
return model, metadata