def test_reset_pop_counters(self):
model = ChathamHouseModel(dict())
model.reset_pop_counters()
assert model.pop_biomass == 0
assert model.pop_nonbiomass == 0
assert model.pop_grid == 0
assert model.pop_offgrid == 0
def testCampModel(self, camptypes, lightingoffgridcost,
elecgriddirectenergy, cookingsolidcost,
lighting_type_descriptions, cooking_type_descriptions):
model = ChathamHouseModel({
'Household Size': 5,
'Electricity Cost': 25,
'Cooking LPG NonCamp Price': 1.8,
'Kerosene CO2 Emissions': 2.96,
'Lighting Offgrid Scaling Factor': 1,
'Cooking Solid Scaling Factor': 1
})
noncampelecgridco2 = {'sdn': 0.615}
camp = 'Southern Darfur : Wilayat - State'
unhcr_camp = {'Southern Darfur : Wilayat - State': (916885, 'sdn')}
population, iso3 = unhcr_camp[camp]
number_hh = model.calculate_number_hh(population)
camp_camptypes = camptypes.get(camp)
elecco2 = noncampelecgridco2[iso3]
tier = 'Baseline'
camplightingoffgridtype = camp_camptypes['Lighting OffGrid %s' % tier]
campcookingsolidtype = camp_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 9.775621570875002
assert offgrid_capital_costs == 4.1443202
assert offgrid_co2_emissions == 15827.062570875001
assert solid_expenditure == 17.732900221858348
assert solid_capital_costs == 0.272694513508602
assert solid_co2_emissions == 486665.6448458868
tier = 'Target 1'
camplightingoffgridtype = camp_camptypes['Lighting OffGrid %s' % tier]
campcookingsolidtype = camp_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 1.5779240444426306
assert offgrid_capital_costs == 8.06913039432437
assert offgrid_co2_emissions == 3835.58834969693
assert solid_expenditure == 7.15870727889462
assert solid_capital_costs == 9.20896466314155
assert solid_co2_emissions == 189409.2845950458
def test_sum_population(self):
remdict = {
'AFG': {
'individual': {
'a': 10,
'b': 20
}
},
'BUR': {
'self-settled': {
'c': 12,
'd': 21
}
}
}
pop = ChathamHouseModel.sum_population(
{'AFG': {
'individual': {
'a': 10,
'b': 20
}
}}, 'AFG', remdict)
assert pop == 30
assert remdict == {
'AFG': {
'individual': {}
},
'BUR': {
'self-settled': {
'c': 12,
'd': 21
}
}
}
def test_calculate_mostfrequent(self):
mostfreq = ChathamHouseModel.calculate_mostfrequent({
'a': 1,
'b': 5,
'c': 5,
'd': 3,
'e': 4
})
assert mostfreq == 5
def test_calculate_regional_average(self):
avg = ChathamHouseModel.calculate_regional_average(
'things', {
'COM': 0.5,
'ETH': 0.1,
'AGO': 0.9
}, 'DJI')
assert avg == (0.3, 14)
avg = ChathamHouseModel.calculate_regional_average(
'things', {
'AGO': 0.3,
'LSO': 0.7,
'DZA': 0.7
}, 'DJI')
assert avg == (0.5, 202)
avg = ChathamHouseModel.calculate_regional_average(
'things', {
'AGO': 0.3,
'COM': 0.5,
'AIA': 0.9
}, 'LBY')
assert avg == (0.4, 2)
def test_get_total_spending(self):
model = ChathamHouseModel(dict())
model.total_spending = 3515.6789123
assert model.get_total_spending() == 3516000000
def test_get_camp_percentage_offgrid(self):
model = ChathamHouseModel(dict())
model.camp_offgrid = 3
model.camp_grid = 2
assert model.get_camp_percentage_offgrid() == 3 / 5
def test_get_percentage_offgrid(self):
model = ChathamHouseModel(dict())
model.total_offgrid = 2
model.total_grid = 3
assert model.get_percentage_offgrid() == 2 / 5
def test_get_camp_percentage_biomass(self):
model = ChathamHouseModel(dict())
model.camp_biomass = 2
model.camp_nonbiomass = 1
assert model.get_camp_percentage_biomass() == 2 / 3
def test_get_percentage_biomass(self):
model = ChathamHouseModel(dict())
model.total_biomass = 1
model.total_nonbiomass = 2
assert model.get_percentage_biomass() == 1 / 3
def test_add_keyfigures(self):
model = ChathamHouseModel(dict())
iso3 = 'AFG'
country = 'Afghanistan'
camp = 'Urban'
tier = 'Baseline'
se = 100
oe = 200
cookingtypedesc = 'Firewood/Charcoal Mix'
cooking_pop = 1000
lightingtypedesc = 'Torch Dependent'
lighting_pop = 2000
results = [list(), None]
ne = 10000
ge = 5000
model.add_keyfigures(iso3,
country,
camp,
tier,
se,
oe,
cookingtypedesc,
cooking_pop,
lightingtypedesc,
lighting_pop,
results,
ne=ne,
ge=ge)
model.reset_pop_counters()
cookingtypedesc = 'LPG/Natural Gas'
cooking_pop = 3500
lightingtypedesc = 'On grid'
lighting_pop = 1500
model.add_keyfigures(iso3,
country,
camp,
tier,
se,
oe,
cookingtypedesc,
cooking_pop,
lightingtypedesc,
lighting_pop,
results,
ne=ne,
ge=ge)
model.reset_pop_counters()
cookingtypedesc = 'Firewood dependent'
cooking_pop = 7000
model.add_keyfigures(iso3,
country,
camp,
tier,
se,
oe,
cookingtypedesc,
cooking_pop,
lightingtypedesc,
lighting_pop,
results,
ne=ne,
ge=ge)
assert results[0] == [[
'AFG', 'Afghanistan', 'Urban', 'Baseline', 10100,
'Firewood/Charcoal Mix', 0, 1000, 5200, 'Torch Dependent', 0, 2000
],
[
'AFG', 'Afghanistan', 'Urban', 'Baseline',
10100, 'LPG/Natural Gas', 3500, 0, 5200,
'On grid', 1500, 0
],
[
'AFG', 'Afghanistan', 'Urban', 'Baseline',
10100, 'Firewood dependent', 0, 7000, 5200,
'On grid', 1500, 0
]]
assert model.get_percentage_biomass() == 8000 / 11500
with pytest.raises(ZeroDivisionError):
model.get_camp_percentage_biomass()
assert model.get_percentage_offgrid() == 2000 / 5000
with pytest.raises(ZeroDivisionError):
model.get_camp_percentage_offgrid()
assert model.get_total_spending() == 45900000000
model.reset_pop_counters()
camp = 'lala'
cookingtypedesc = 'Alternative biomass'
lightingtypedesc = 'Kerosene dependent'
model.add_keyfigures(iso3, country, camp, tier, se, oe,
cookingtypedesc, cooking_pop, lightingtypedesc,
lighting_pop, results)
assert results[0] == [[
'AFG', 'Afghanistan', 'Urban', 'Baseline', 10100,
'Firewood/Charcoal Mix', 0, 1000, 5200, 'Torch Dependent', 0, 2000
],
[
'AFG', 'Afghanistan', 'Urban', 'Baseline',
10100, 'LPG/Natural Gas', 3500, 0, 5200,
'On grid', 1500, 0
],
[
'AFG', 'Afghanistan', 'Urban', 'Baseline',
10100, 'Firewood dependent', 0, 7000, 5200,
'On grid', 1500, 0
],
[
'AFG', 'Afghanistan', 'lala', 'Baseline',
100, 'Alternative biomass', 7000, 0, 200,
'Kerosene dependent', 0, 1500
]]
assert model.get_total_spending() == 46200000000
assert model.get_percentage_biomass() == 8000 / 18500
assert model.get_camp_percentage_biomass() == 0
assert model.get_percentage_offgrid() == 3500 / 6500
assert model.get_camp_percentage_offgrid() == 1500 / 1500
def testSmallCampModel(self, small_camptypes, lightingoffgridcost,
elecgriddirectenergy, cookingsolidcost,
lighting_type_descriptions,
cooking_type_descriptions):
model = ChathamHouseModel({
'Household Size': 5,
'Electricity Cost': 25,
'Cooking LPG NonCamp Price': 1.8,
'Kerosene CO2 Emissions': 2.96,
'Lighting Offgrid Scaling Factor': 1,
'Cooking Solid Scaling Factor': 1
})
camp_group = 'Subsaharan Africa G'
smallcamps = {camp_group: 83908.56}
small_camps_elecgridco2 = {camp_group: 0.3994098361}
number_hh = model.calculate_number_hh(smallcamps[camp_group])
campgroup_camptypes = small_camptypes.get(camp_group)
elecco2 = small_camps_elecgridco2[camp_group]
tier = 'Baseline'
camplightingoffgridtype = campgroup_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = campgroup_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 0.894614187294
assert offgrid_capital_costs == 0.37926669120000006
assert offgrid_co2_emissions == 1442.1696815239604
assert solid_expenditure == ''
assert solid_capital_costs == ''
assert solid_co2_emissions == ''
tier = 'Target 3'
camplightingoffgridtype = campgroup_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = campgroup_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 0.15505425729584385
assert offgrid_capital_costs == 8.547099603928343
assert offgrid_co2_emissions == 592.1439284539138
assert solid_expenditure == ''
assert solid_capital_costs == ''
assert solid_co2_emissions == ''
camp_group = 'Asia E'
smallcamps = {camp_group: 19577.01373}
small_camps_elecgridco2 = {camp_group: ''}
number_hh = model.calculate_number_hh(smallcamps[camp_group])
campgroup_camptypes = small_camptypes.get(camp_group)
elecco2 = small_camps_elecgridco2[camp_group]
tier = 'Target 2'
camplightingoffgridtype = campgroup_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = campgroup_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == ''
assert offgrid_capital_costs == ''
assert offgrid_co2_emissions == ''
assert solid_expenditure == 0.28453285405596584
assert solid_capital_costs == 0.19577013729999998
assert solid_co2_emissions == 520.506277903452
tier = 'Baseline'
camplightingoffgridtype = campgroup_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = campgroup_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost, elecgriddirectenergy,
elecco2, number_hh, cooking_type_descriptions,
campcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == ''
assert offgrid_capital_costs == ''
assert offgrid_co2_emissions == ''
assert solid_expenditure == 0.6505642389516176
assert solid_capital_costs == 0.00916640744582953
assert solid_co2_emissions == 11087.728057822153
def testNonCampModel(self, lightingoffgridcost, elecgriddirectenergy,
cookingsolidcost, slumratios,
lighting_type_descriptions,
cooking_type_descriptions):
model = ChathamHouseModel({
'Population Adjustment Factor': 0.7216833622,
'Household Size': 5,
'Electricity Cost': 25,
'Cooking LPG NonCamp Price': 1.8,
'Kerosene CO2 Emissions': 2.96,
'Lighting Offgrid Scaling Factor': 1,
'Cooking Solid Scaling Factor': 1
})
iso3 = 'ago'
unhcr_non_camp = {iso3: 59970}
urbanratios = {iso3: 0.58379}
elecappliances = {iso3: 92.6033836492}
noncampelecgridco2 = {iso3: 0.0375}
cookinglpg = {iso3: 4.096473669}
country_elecappliances = elecappliances.get(iso3)
country_noncampelecgridco2 = noncampelecgridco2[iso3]
country_cookinglpg = cookinglpg[iso3]
displaced_population = unhcr_non_camp[iso3]
number_hh_by_pop_type = model.calculate_population(
iso3, displaced_population, urbanratios, slumratios, list())
assert number_hh_by_pop_type == {
'Rural': 1389.3629848179437,
'Slum': 7601.16815388216,
'Urban': 3003.4688612998957
}
number_hh_by_pop_type = model.calculate_population(
iso3, displaced_population, urbanratios, {iso3: 0.658}, list())
assert number_hh_by_pop_type == {
'Rural': 1389.3629848179437,
'Slum': 6977.851155989793,
'Urban': 3626.785859192263
}
pop_type = 'Rural'
number_hh = number_hh_by_pop_type[pop_type]
country_elec_access = 0.055
hh_grid_access, hh_offgrid = model.calculate_hh_access(
number_hh, country_elec_access)
assert hh_grid_access == 76.4149641649869
assert hh_offgrid == 1312.9480206529568
country_nonsolid_access = 0.11
hh_nonsolid_access, hh_no_nonsolid_access = model.calculate_hh_access(
number_hh, country_nonsolid_access)
assert hh_nonsolid_access == 152.8299283299738
assert hh_no_nonsolid_access == 1236.5330564879698
elecgrid = {0: 3, 1: 35, 2: 194, 3: 820, 4: 1720}
grid_expenditure, grid_co2_emissions = model.calculate_ongrid_lighting(
hh_grid_access, elecgrid, country_elecappliances,
country_noncampelecgridco2)
assert grid_expenditure == 0.0017690710607775378
assert grid_co2_emissions == 0.26536065911663065
nonsolid_expenditure, nonsolid_co2_emissions = model.calculate_non_solid_cooking(
hh_nonsolid_access, country_cookinglpg)
assert nonsolid_expenditure == 0.013522977588360128
assert nonsolid_co2_emissions == 22.237785367525543
noncamplightingoffgridtypes = {
'Urban Baseline Type': 1,
'Urban Target 1 Type': 1,
'Urban Target 2 Type': 3,
'Urban Target 3 Type': 7,
'Rural Baseline Type': 1,
'Rural Target 1 Type': 1,
'Rural Target 2 Type': 3,
'Rural Target 3 Type': 7,
'Slum Baseline Type': 1,
'Slum Target 1 Type': 1,
'Slum Target 2 Type': 3,
'Slum Target 3 Type': 7
}
noncampcookingsolidtypes = {
'Urban Baseline Type': 2,
'Urban Target 1 Type': 2,
'Urban Target 2 Type': 7,
'Urban Target 3 Type': 8,
'Rural Baseline Type': 1,
'Rural Target 1 Type': 1,
'Rural Target 2 Type': 7,
'Rural Target 3 Type': 8,
'Slum Baseline Type': 1,
'Slum Target 1 Type': 1,
'Slum Target 2 Type': 7,
'Slum Target 3 Type': 8
}
tier = 'Target 2'
noncamplightingoffgridtype = noncamplightingoffgridtypes['%s %s Type' %
(pop_type,
tier)]
noncampcookingsolidtype = noncampcookingsolidtypes['%s %s Type' %
(pop_type, tier)]
res = model.calculate_offgrid_solid(
tier, hh_offgrid, lighting_type_descriptions,
noncamplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_noncampelecgridco2,
hh_no_nonsolid_access, cooking_type_descriptions,
noncampcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 0.002157308870967376
assert offgrid_capital_costs == 0.2328784963573492
assert offgrid_co2_emissions == 8.238647374164904
co2_emissions = grid_co2_emissions + offgrid_co2_emissions
assert co2_emissions == 8.504008033281535
assert solid_expenditure == 0.09999499528651126
assert solid_capital_costs == 0.06182665282439849
assert solid_co2_emissions == 1405.3438532905193
co2_emissions = nonsolid_co2_emissions + solid_co2_emissions
assert co2_emissions == 1427.5816386580448
tier = 'Target 3'
noncamplightingoffgridtype = noncamplightingoffgridtypes['%s %s Type' %
(pop_type,
tier)]
noncampcookingsolidtype = noncampcookingsolidtypes['%s %s Type' %
(pop_type, tier)]
res = model.calculate_offgrid_solid(
tier, hh_offgrid, lighting_type_descriptions,
noncamplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_noncampelecgridco2,
hh_no_nonsolid_access, cooking_type_descriptions,
noncampcookingsolidtype, cookingsolidcost)
lighting_type_description, offgrid_expenditure, offgrid_capital_costs, offgrid_co2_emissions, \
cooking_type_description, solid_expenditure, solid_capital_costs, solid_co2_emissions = res
assert offgrid_expenditure == 0.01213095423222568
assert offgrid_capital_costs == 0.6686980152740906
assert offgrid_co2_emissions == 46.32746639944912
co2_emissions = grid_co2_emissions + offgrid_co2_emissions
assert co2_emissions == 46.59282705856575
assert solid_expenditure == 0.3275797640995024
assert solid_capital_costs == 0.0989177392294985
assert solid_co2_emissions == 290.5365064344328
co2_emissions = nonsolid_co2_emissions + solid_co2_emissions
assert co2_emissions == 312.7742918019583
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
with Download() as downloader:
constants = float_value_convert(
downloader.download_tabular_key_value(
configuration['constants_url']))
constants['Lighting Grid Tier'] = int(constants['Lighting Grid Tier'])
camp_overrides = downloader.download_tabular_cols_as_dicts(
configuration['camp_overrides_url'])
camp_overrides['Population'] = integer_value_convert(
camp_overrides['Population'], dropfailedvalues=True)
camp_overrides['Country'] = key_value_convert(
camp_overrides['Country'], valuefn=get_iso3)
datasets = Dataset.search_in_hdx('displacement',
fq='organization:unhcr')
all_camps_per_country, unhcr_non_camp, unhcr_camp, unhcr_camp_excluded = \
get_camp_non_camp_populations(constants['Non Camp Types'], constants['Camp Types'],
camp_overrides, datasets, downloader)
country_totals = copy.deepcopy(all_camps_per_country)
world_bank_url = configuration['world_bank_url']
urbanratios = get_worldbank_series(
world_bank_url % configuration['urban_ratio_wb'], downloader)
slumratios = get_slumratios(configuration['slum_ratio_url'],
downloader)
noncamp_elec_access = dict()
noncamp_elec_access['Urban'] = get_worldbank_series(
world_bank_url % configuration['urban_elec_wb'], downloader)
noncamp_elec_access['Rural'] = get_worldbank_series(
world_bank_url % configuration['rural_elec_wb'], downloader)
noncamp_elec_access['Slum'] = avg_dicts(noncamp_elec_access['Rural'],
noncamp_elec_access['Urban'])
ieadata = downloader.download_tabular_cols_as_dicts(
configuration['iea_data_url'])
elecappliances = key_value_convert(ieadata['Electrical Appliances'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
cookinglpg = key_value_convert(ieadata['Cooking LPG'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
elecgridtiers = key_value_convert(
downloader.download_tabular_key_value(
configuration['elec_grid_tiers_url']),
keyfn=int,
valuefn=float)
elecgriddirectenergy = float_value_convert(
downloader.download_tabular_key_value(
configuration['elec_grid_direct_energy_url']))
elecgridco2 = key_value_convert(downloader.download_tabular_key_value(
configuration['elec_grid_co2_url']),
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
def get_elecgridco2(iso, inf):
elgridco2 = elecgridco2.get(iso)
if elgridco2 is None:
elgridco2, reg = model.calculate_regional_average(
'Grid CO2', elecgridco2, iso)
inf.append('elco2(%s)=%.3g' % (reg, elgridco2))
return elgridco2
noncamptypes = downloader.download_tabular_cols_as_dicts(
configuration['noncamp_types_url'])
noncamplightingoffgridtypes = integer_value_convert(
noncamptypes['Lighting OffGrid'])
noncampcookingsolidtypes = integer_value_convert(
noncamptypes['Cooking Solid'])
camptypes = get_camptypes(configuration['camp_types_url'], downloader)
camptypes_fallbacks_offgrid, camptypes_fallbacks_solid = \
get_camptypes_fallbacks(configuration['camp_types_fallbacks_url'], downloader, keyfn=get_iso3)
costs = downloader.download_tabular_cols_as_dicts(
configuration['costs_url'])
lightingoffgridcost = float_value_convert(costs['Lighting OffGrid'])
cookingsolidcost = float_value_convert(costs['Cooking Solid'])
noncamp_nonsolid_access = downloader.download_tabular_cols_as_dicts(
configuration['noncamp_cooking_nonsolid_url'])
noncamp_nonsolid_access['Urban'] = key_value_convert(
noncamp_nonsolid_access['Urban'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
noncamp_nonsolid_access['Rural'] = key_value_convert(
noncamp_nonsolid_access['Rural'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
noncamp_nonsolid_access['Slum'] = noncamp_nonsolid_access['Urban']
small_camptypes = get_camptypes(configuration['small_camptypes_url'],
downloader)
small_camp_data = downloader.download_tabular_cols_as_dicts(
configuration['small_camps_data_url'])
smallcamps = float_value_convert(small_camp_data['Population'])
small_camps_elecgridco2 = float_value_convert(
small_camp_data['Electricity Grid CO2'])
type_descriptions = downloader.download_tabular_cols_as_dicts(
configuration['type_descriptions_url'])
lighting_type_descriptions = type_descriptions['Lighting Descriptions']
cooking_type_descriptions = type_descriptions['Cooking Descriptions']
model = ChathamHouseModel(constants)
pop_types = ['Urban', 'Slum', 'Rural', 'Camp', 'Small Camp']
headers = list()
results = list()
for i, pop_type in enumerate(pop_types):
results.append(list())
if pop_type == 'Camp':
headers.append(['ISO3 Country Code', 'Country Name', 'Camp Name'])
hxlheaders = ['#country+code', '#country+name', '#loc+name']
elif pop_type == 'Small Camp':
headers.append(['Region'])
hxlheaders = ['#region+name']
else:
headers.append(['ISO3 Country Code', 'Country Name'])
hxlheaders = ['#country+code', '#country+name']
headers[-1].extend(['Population', 'Tier'])
hxlheaders.extend(['#population+num', '#indicator+tier'])
if pop_type not in ['Camp', 'Small Camp']:
headers[-1].extend(
['Grid Expenditure ($m/yr)', 'Grid CO2 Emissions (t/yr)'])
hxlheaders.extend([
'#indicator+value+grid+expenditure',
'#indicator+value+grid+co2_emissions'
])
headers[-1].extend([
'Offgrid Type', 'Lighting Type Description',
'Offgrid Expenditure ($m/yr)', 'Offgrid Capital Costs ($m)',
'Offgrid CO2 Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+type+offgrid', '#indicator+text+lighting',
'#indicator+value+offgrid+expenditure',
'#indicator+value+offgrid+capital_costs',
'#indicator+value+offgrid+co2_emissions'
])
if pop_type not in ['Camp', 'Small Camp']:
headers[-1].extend([
'Nonsolid Expenditure ($m/yr)', 'Nonsolid CO2 Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+value+nonsolid+expenditure',
'#indicator+value+nonsolid+co2_emissions'
])
headers[-1].extend([
'Solid Type', 'Cooking Type Description',
'Solid Expenditure ($m/yr)', 'Solid Capital Costs ($m)',
'Solid CO2_Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+type+solid', '#indicator+text+cooking',
'#indicator+value+solid+expenditure',
'#indicator+value+solid+capital_costs',
'#indicator+value+solid+co2_emissions'
])
if pop_type != 'Small Camp':
headers[-1].append('Info')
hxlheaders.append('#meta+info')
results[i].append(hxlheaders)
results.append(list())
headers.append(['ISO3 Country Code', 'Country Name', 'Population'])
hxlheaders = ['#country+code', '#country+name', '#population+num']
results[len(results) - 1].append(hxlheaders)
results.append(list())
headers.append([
'ISO3 Country Code', 'Country Name', 'Camp', 'Tier',
'Cooking Spending', 'Cooking Description',
'Population not using Biomass', 'Population using Biomass',
'Lighting Spending', 'Lighting Description', 'Population on Grid',
'Population off Grid'
])
results.append(list())
headers.append([
'code', 'title', 'value', 'latest_date', 'source', 'source_link',
'notes', 'explore', 'units'
])
today = datetime.utcnow()
for iso3 in sorted(unhcr_non_camp):
info = list()
population = model.sum_population(unhcr_non_camp, iso3,
all_camps_per_country)
number_hh_by_pop_type = model.calculate_population(
iso3, population, urbanratios, slumratios, info)
country_elecappliances = elecappliances.get(iso3)
if country_elecappliances is None:
country_elecappliances, region = \
model.calculate_regional_average('Electrical Appliances', elecappliances, iso3)
info.append('elap(%s)=%.3g' % (region, country_elecappliances))
country_elecgridco2 = get_elecgridco2(iso3, info)
country_cookinglpg = cookinglpg.get(iso3)
if country_cookinglpg is None:
country_cookinglpg, region = model.calculate_regional_average(
'LPG', cookinglpg, iso3)
info.append('lpg(%s)=%.3g' % (region, country_elecappliances))
cn = Country.get_country_name_from_iso3(iso3)
for pop_type in number_hh_by_pop_type:
model.reset_pop_counters()
info2 = copy.deepcopy(info)
number_hh = number_hh_by_pop_type[pop_type]
country_elec_access = noncamp_elec_access[pop_type].get(iso3)
if country_elec_access is None:
country_elec_access, region = \
model.calculate_regional_average('Grid access', noncamp_elec_access[pop_type], iso3)
info2.append('elac(%s)=%.3g' %
(region, country_elecappliances))
hh_grid_access, hh_offgrid = model.calculate_hh_access(
number_hh, country_elec_access)
pop_grid_access = model.calculate_population_from_hh(
hh_grid_access)
pop_offgrid_access = model.calculate_population_from_hh(hh_offgrid)
model.pop_grid += pop_grid_access
country_noncamp_nonsolid_access = noncamp_nonsolid_access[
pop_type].get(iso3)
if country_noncamp_nonsolid_access is None:
country_noncamp_nonsolid_access, region = \
model.calculate_regional_average('Nonsolid access', noncamp_nonsolid_access[pop_type], iso3)
info2.append('nsac(%s)=%.3g' %
(region, country_elecappliances))
hh_nonsolid_access, hh_no_nonsolid_access = \
model.calculate_hh_access(number_hh, country_noncamp_nonsolid_access)
pop_biomass_access = model.calculate_population_from_hh(
hh_no_nonsolid_access)
pop_nonbiomass_access = model.calculate_population_from_hh(
hh_nonsolid_access)
model.pop_nonbiomass += pop_nonbiomass_access
ge, gc = model.calculate_ongrid_lighting(hh_grid_access,
elecgridtiers,
country_elecappliances,
country_elecgridco2)
ne, nc = model.calculate_non_solid_cooking(hh_nonsolid_access,
country_cookinglpg)
for tier in model.tiers:
info3 = copy.deepcopy(info2)
noncamplightingoffgridtype = model.get_noncamp_type(
noncamplightingoffgridtypes, pop_type, tier)
noncampcookingsolidtype = model.get_noncamp_type(
noncampcookingsolidtypes, pop_type, tier)
res = model.calculate_offgrid_solid(
tier, hh_offgrid, lighting_type_descriptions,
noncamplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2,
hh_no_nonsolid_access, cooking_type_descriptions,
noncampcookingsolidtype, cookingsolidcost)
noncamplightingtypedesc, oe, oc, oco2, noncampcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures(iso3,
cn,
pop_type,
tier,
se,
oe,
noncampcookingtypedesc,
pop_biomass_access,
noncamplightingtypedesc,
pop_offgrid_access,
results,
ne=ne,
ge=ge)
population = model.calculate_population_from_hh(number_hh)
info3 = ','.join(info3)
row = [
iso3, cn, population, tier, ge, gc,
noncamplightingoffgridtype, noncamplightingtypedesc, oe,
oc, oco2, ne, nc, noncampcookingsolidtype,
noncampcookingtypedesc, se, sc, sco2, info3
]
results[pop_types.index(pop_type.capitalize())].append(row)
camp_offgridtypes_in_countries = dict()
camp_solidtypes_in_countries = dict()
missing_from_unhcr = list()
for name in sorted(camptypes):
model.reset_pop_counters()
info = list()
unhcrcampname = name
result = unhcr_camp.get(unhcrcampname)
if result is None:
firstpart = name.split(':')[0].strip()
for unhcrcampname in sorted(unhcr_camp):
if firstpart in unhcrcampname:
result = unhcr_camp[unhcrcampname]
logger.info(
'Matched first part of name of %s to UNHCR name: %s' %
(name, unhcrcampname))
info.append('Matched %s' % firstpart)
break
if result is None:
camptype = unhcr_camp_excluded.get(name)
if camptype is None:
if check_name_dispersed(name):
logger.info(
'Camp %s from the spreadsheet has been treated as non-camp!'
% name)
else:
missing_from_unhcr.append(name)
else:
logger.info('Camp %s is in UNHCR data but has camp type %s!' %
(name, camptype))
continue
population, iso3, accommodation_type = result
del all_camps_per_country[iso3][accommodation_type][unhcrcampname]
camp_camptypes = camptypes[name]
number_hh = model.calculate_number_hh(population)
country_elecgridco2 = get_elecgridco2(iso3, info)
for tier in model.tiers:
info2 = copy.deepcopy(info)
camplightingoffgridtype = camp_camptypes.get(
'Lighting OffGrid %s' % tier)
if camplightingoffgridtype is None:
logger.warning('No Lighting OffGrid %s for %s in %s' %
(tier, name, cn))
campcookingsolidtype = camp_camptypes.get('Cooking Solid %s' %
tier)
if campcookingsolidtype is None:
logger.warning('No Cooking Solid %s for %s in %s' %
(tier, name, cn))
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
cn = Country.get_country_name_from_iso3(iso3)
model.add_keyfigures(iso3, cn, name, tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population, results)
info2 = ','.join(info2)
row = [
iso3, cn, name, population, tier, camplightingoffgridtype,
camplightingtypedesc, oe, oc, oco2, campcookingsolidtype,
campcookingtypedesc, se, sc, sco2, info2
]
results[pop_types.index('Camp')].append(row)
if camplightingoffgridtype:
append_value(camp_offgridtypes_in_countries, iso3, tier, name,
camplightingoffgridtype)
if campcookingsolidtype:
append_value(camp_solidtypes_in_countries, iso3, tier, name,
campcookingsolidtype)
logger.info(
'The following camps are in the spreadsheet but not in the UNHCR data : %s'
% ', '.join(missing_from_unhcr))
for iso3 in sorted(country_totals):
info = list()
population = model.sum_population(country_totals, iso3)
cn = Country.get_country_name_from_iso3(iso3)
row = [iso3, cn, population]
results[len(results) - 3].append(row)
extra_camp_types = all_camps_per_country[iso3]
country_elecgridco2 = get_elecgridco2(iso3, info)
for accommodation_type in sorted(extra_camp_types):
camps = extra_camp_types[accommodation_type]
for name in sorted(camps):
model.reset_pop_counters()
info2 = copy.deepcopy(info)
population = camps[name]
if population < 20000:
logger.info(
'Ignoring extra camp %s from UNHCR data with population %s (<20000) and accommodation type %s in country %s.'
% (name, population, accommodation_type, cn))
continue
number_hh = model.calculate_number_hh(population)
offgrid_tiers_in_country = camp_offgridtypes_in_countries.get(
iso3)
if offgrid_tiers_in_country is None:
offgrid_tiers_in_country = camptypes_fallbacks_offgrid.get(
iso3)
if not offgrid_tiers_in_country:
logger.warning(
'Missing fallback for country %s, where UNHCR data has extra camp %s with population %s and accommodation type %s'
% (cn, name, population, accommodation_type))
continue
info2.append('UNHCR only')
for tier in offgrid_tiers_in_country:
info3 = copy.deepcopy(info2)
camplightingoffgridtype = offgrid_tiers_in_country[tier]
if isinstance(camplightingoffgridtype, int):
campcookingsolidtype = camptypes_fallbacks_solid[iso3][
tier]
info3.append('Fallback')
else:
camplightingoffgridtype = model.calculate_mostfrequent(
offgrid_tiers_in_country[tier])
campcookingsolidtype = model.calculate_mostfrequent(
camp_solidtypes_in_countries[iso3][tier])
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures(iso3, cn, name, tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population,
results)
info3 = ','.join(info3)
row = [
iso3, cn, name, population, tier,
camplightingoffgridtype, camplightingtypedesc, oe, oc,
oco2, campcookingsolidtype, campcookingtypedesc, se,
sc, sco2, info3
]
results[pop_types.index('Camp')].append(row)
for region in sorted(smallcamps):
model.reset_pop_counters()
info = list()
population = smallcamps[region]
if not population or population == '-':
continue
number_hh = model.calculate_number_hh(population)
region_camptypes = small_camptypes.get(region)
if region_camptypes is None:
logger.info('Missing camp group %s in small camp types!' % region)
continue
elecco2 = small_camps_elecgridco2[region]
if not elecco2 or elecco2 == '-':
info.append('Blank elco2')
elecco2 = 0
for tier in model.tiers:
info2 = copy.deepcopy(info)
camplightingoffgridtype = region_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = region_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, elecco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures('', region, 'small camp', tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population, results)
info2 = ','.join(info2)
row = [
region,
model.round(population), tier, camplightingoffgridtype,
camplightingtypedesc, oe, oc, oco2, campcookingsolidtype,
campcookingtypedesc, se, sc, sco2, info2
]
results[pop_types.index('Small Camp')].append(row)
date = today.date().isoformat()
source = 'Estimate from the Moving Energy Initiative'
data_url = 'https://data.humdata.org/dataset/energy-consumption-of-refugees-and-displaced-people'
rows = [
[
'MEI01',
'% of Refugees and Displaced People Cooking with Biomass in Camps',
model.get_camp_percentage_biomass(), date, source, data_url, '',
'', 'ratio'
],
[
'MEI02', '% of Refugees and Displaced People Off-Grid in Camps',
model.get_camp_percentage_offgrid(), date, source, data_url, '',
'', 'ratio'
],
[
'MEI03',
'Total Annual Energy Spending by Refugees and Displaced People',
model.get_total_spending(), date, source, data_url, '', '',
'dollars_million'
],
[
'MEI04', 'No. of Countries Hosting Refugees and Displaced People',
len(country_totals), date, source, data_url, '', '', 'count'
]
]
results[len(results) - 1].extend(rows)
dataset, resources, showcase = generate_dataset_resources_and_showcase(
pop_types, today)
folder = gettempdir()
file_to_upload = None
for i, _ in enumerate(results):
resource = resources[i]
file_to_upload = join(folder, resource['name'])
write_list_to_csv(results[i], file_to_upload, headers=headers[i])
resource.set_file_to_upload(file_to_upload)
dataset.add_update_resources(resources)
dataset.update_from_yaml()
# dataset.create_in_hdx()
for resource in dataset.get_resources():
name = resource['name'].lower()
if 'figures' in name and 'disagg' not in name:
logger.info('Updating key figures datastore for %s' % name)