def test_read_write_list_to_csv(self):
l = [[1, 2, 3, 'a'], [4, 5, 6, 'b'], [7, 8, 9, 'c']]
folder = gettempdir()
filename = 'test_read_write_list_to_csv.csv'
filepath = join(folder, filename)
write_list_to_csv(l, filepath, headers=['h1', 'h2', 'h3', 'h4'])
newll = read_list_from_csv(filepath)
newld = read_list_from_csv(filepath, dict_form=True, headers=1)
remove(filepath)
assert newll == [['h1', 'h2', 'h3', 'h4'], ['1', '2', '3', 'a'],
['4', '5', '6', 'b'], ['7', '8', '9', 'c']]
assert newld == [{
'h1': '1',
'h2': '2',
'h4': 'a',
'h3': '3'
}, {
'h1': '4',
'h2': '5',
'h4': 'b',
'h3': '6'
}, {
'h1': '7',
'h2': '8',
'h4': 'c',
'h3': '9'
}]
def update_cbpf(base_url, downloader, poolfundabbrv, today, valid_names,
replace_values, resource_updates):
year = today.year
if today.month <= 3:
year -= 1
response = downloader.download('%sProjectSummary?poolfundAbbrv=%s' %
(base_url, poolfundabbrv))
jsonresponse = response.json()
projects = jsonresponse['value']
transactions = dict()
for project in projects:
if project['AllocationYear'] != year:
continue
code = project['ChfProjectCode']
budget = project['Budget']
directcost = project['TotalDirectCost']
supportcost = project['TotalSupportCost']
transactions[code] = budget, directcost, supportcost
response = downloader.download('%sLocation?poolfundAbbrv=%s' %
(base_url, poolfundabbrv))
jsonresponse = response.json()
locations = jsonresponse['value']
totals = dict()
for location in locations:
if location['AllocationYear'] != year:
continue
code = location['ChfProjectCode']
admin1 = cannonize_name(location['AdminLocation1'], valid_names,
replace_values)
percentage = float(location['Percentage']) / 100.0
budget, directcost, supportcost = transactions[code]
totalbudget, totaldirectcost, totalsupportcost = totals.get(
admin1, (0.0, 0.0, 0.0))
budget *= percentage
directcost *= percentage
supportcost *= percentage
totalbudget += budget
totaldirectcost += directcost
totalsupportcost += supportcost
totals[admin1] = totalbudget, totaldirectcost, totalsupportcost
rows = list()
rows.append(['#adm1+name', '#cashflow+type', '#cashflow+value'])
for admin1 in sorted(totals):
budget, directcost, supportcost = totals[admin1]
rows.append([admin1, 'Budget', round(budget)])
rows.append([admin1, 'Direct Cost', round(directcost)])
rows.append([admin1, 'Support Cost', round(supportcost)])
write_list_to_csv(
rows,
resource_updates['cbpf']['path'],
headers=['Admin Location', 'Cashflow Type', 'Cashflow Value'])
def output_csv():
if rows is None:
return
headers = deepcopy(downloader.response.headers)
for i, header in enumerate(headers):
if 'year' in header.lower():
headers.insert(i, 'EndYear')
headers.insert(i, 'StartYear')
break
headers.insert(0, 'Iso3')
hxlrow = dict()
for header in headers:
hxlrow[header] = hxltags.get(header, '')
rows.insert(0, hxlrow)
filepath = join(folder, '%s_%s.csv' % (indicatorname, countrycode))
write_list_to_csv(rows, filepath, headers=headers)
ds = datasets[-1]
ds.set_dataset_year_range(earliest_year, latest_year)
ds.resources[0].set_file_to_upload(filepath)
def test_read_write_list_to_csv(self):
list_of_lists = [[1, 2, 3, "a"], [4, 5, 6, "b"], [7, 8, 9, "c"]]
folder = gettempdir()
filename = "test_read_write_list_to_csv.csv"
filepath = join(folder, filename)
write_list_to_csv(filepath,
list_of_lists,
headers=["h1", "h2", "h3", "h4"])
newll = read_list_from_csv(filepath)
newld = read_list_from_csv(filepath, headers=1, dict_form=True)
remove(filepath)
assert newll == [
["h1", "h2", "h3", "h4"],
["1", "2", "3", "a"],
["4", "5", "6", "b"],
["7", "8", "9", "c"],
]
assert newld == [
{
"h1": "1",
"h2": "2",
"h4": "a",
"h3": "3"
},
{
"h1": "4",
"h2": "5",
"h4": "b",
"h3": "6"
},
{
"h1": "7",
"h2": "8",
"h4": "c",
"h3": "9"
},
]
with pytest.raises(ValueError):
read_list_from_csv(filepath, dict_form=True)
def get_fts(basic_auths, configuration, countryiso3s, scrapers=None):
name = inspect.currentframe().f_code.co_name
if scrapers and not any(scraper in name for scraper in scrapers):
return list(), list(), list(), list(), list(), list()
hrp_requirements = dict()
hrp_funding = dict()
hrp_percentage = dict()
hrp_covid_requirements = dict()
hrp_covid_funding = dict()
hrp_covid_percentage = dict()
hrp_gbv_funding = dict()
other_planname = dict()
other_requirements = dict()
other_funding = dict()
other_percentage = dict()
def add_other_requirements_and_funding(iso3, name, req, fund, pct):
dict_of_lists_add(other_planname, iso3, name)
if req:
dict_of_lists_add(other_requirements, iso3, req)
else:
dict_of_lists_add(other_requirements, iso3, None)
if fund and req:
dict_of_lists_add(other_funding, iso3, fund)
dict_of_lists_add(other_percentage, iso3, pct)
else:
dict_of_lists_add(other_funding, iso3, None)
dict_of_lists_add(other_percentage, iso3, None)
fts_configuration = configuration['fts']
v1_url = fts_configuration['v1_url']
v2_url = fts_configuration['v2_url']
total_covidreq = 0
total_gbvfund = 0
rows = list()
def add_covid_gbv_requirements_and_funding(name,
includetotals,
req,
fund,
gbvfund=None):
nonlocal total_covidreq, total_gbvfund
if not includetotals:
return
if req or fund or gbvfund:
rows.append([name, req, fund, gbvfund])
if req:
logger.info('%s: Requirements=%d' % (name, req))
total_covidreq += req
if fund:
logger.info('%s: Funding=%d' % (name, fund))
if gbvfund:
logger.info('%s: GBV Funding=%d' % (name, gbvfund))
total_gbvfund += gbvfund
with Download(basic_auth=basic_auths.get('fts'),
rate_limit={
'calls': 1,
'period': 1
}) as downloader:
url = '%sfts/flow/plan/overview/progress/%d' % (v2_url, today.year)
data = download_data(url, downloader)
plans = data['plans']
plan_ids = ','.join([str(plan['id']) for plan in plans])
url = '%sfts/flow?emergencyid=911&planid=%s&groupby=plan' % (v1_url,
plan_ids)
funding_data = download_data(url, downloader)
fundingtotals = funding_data['report3']['fundingTotals']
total_covidfund = fundingtotals['total']
fundingobjects = fundingtotals['objects']
for plan in plans:
plan_id = plan['id']
planname = plan['name']
allreq = plan['requirements']['revisedRequirements']
funding = plan.get('funding')
if funding:
allfund = funding['totalFunding']
else:
allfund = None
includetotals = plan['planType']['includeTotals']
gbvfund = get_gbv_funding(v1_url, plan_id, downloader)
if plan_id == 952:
add_covid_gbv_requirements_and_funding(planname, includetotals,
allreq, allfund,
gbvfund)
continue
covidreq, covidfund = get_requirements_and_funding(
v1_url, v2_url, plan_id, downloader, fundingobjects)
add_covid_gbv_requirements_and_funding(planname, includetotals,
covidreq, covidfund,
gbvfund)
countries = plan['countries']
countryid_iso3mapping = dict()
for country in countries:
countryiso = country['iso3']
if countryiso:
countryid = country['id']
countryid_iso3mapping[countryid] = countryiso
if len(countryid_iso3mapping) == 0:
continue
if len(countryid_iso3mapping) == 1:
countryiso = countryid_iso3mapping.popitem()[1]
if not countryiso or countryiso not in countryiso3s:
continue
plan_type = plan['planType']['name'].lower()
if funding:
allpct = get_fraction_str(funding['progress'], 100)
else:
allpct = None
if plan_type == 'humanitarian response plan':
if allreq:
hrp_requirements[countryiso] = allreq
else:
hrp_requirements[countryiso] = None
if allfund and allreq:
hrp_funding[countryiso] = allfund
hrp_percentage[countryiso] = allpct
if covidreq:
hrp_covid_requirements[countryiso] = covidreq
else:
hrp_covid_requirements[countryiso] = None
if covidfund and covidreq:
hrp_covid_funding[countryiso] = covidfund
hrp_covid_percentage[countryiso] = get_fraction_str(
covidfund, covidreq)
if gbvfund:
hrp_gbv_funding[countryiso] = gbvfund
else:
planname = map_planname(planname)
add_other_requirements_and_funding(countryiso, planname,
allreq, allfund, allpct)
else:
allreqs, allfunds = get_requirements_and_funding_location(
v1_url, plan_id, countryid_iso3mapping, countryiso3s,
downloader)
planname = map_planname(planname)
for countryiso in allreqs:
allreq = allreqs[countryiso]
allfund = allfunds.get(countryiso)
if allfund:
allpct = get_fraction_str(allfund, allreq)
else:
allpct = None
add_other_requirements_and_funding(countryiso, planname,
allreq, allfund, allpct)
def create_output(vallist):
strings = list()
for val in vallist:
if val is None:
strings.append('')
else:
strings.append(str(val))
return '|'.join(strings)
for countryiso in other_requirements:
other_planname[countryiso] = create_output(
other_planname[countryiso])
other_requirements[countryiso] = create_output(
other_requirements[countryiso])
other_funding[countryiso] = create_output(
other_funding[countryiso])
other_percentage[countryiso] = create_output(
other_percentage[countryiso])
total_allreq = data['totals']['revisedRequirements']
total_allfund = data['totals']['totalFunding']
total_allpercent = get_fraction_str(data['totals']['progress'], 100)
total_covidpercent = get_fraction_str(total_covidfund, total_covidreq)
logger.info('Processed FTS')
write_list_to_csv('ftscovid.csv', rows,
['Name', 'Requirements', 'Funding'])
ghxltags = [
'#value+funding+hrp+required+usd', '#value+funding+hrp+total+usd',
'#value+funding+hrp+pct', '#value+covid+funding+hrp+required+usd',
'#value+covid+funding+hrp+total+usd',
'#value+covid+funding+hrp+pct', '#value+funding+gbv+hrp+total+usd'
]
hxltags = ghxltags + [
'#value+funding+other+planname',
'#value+funding+other+required+usd',
'#value+funding+other+total+usd', '#value+funding+other+pct'
]
total_allreq = {'global': total_allreq}
total_allfund = {'global': total_allfund}
total_allpercent = {'global': total_allpercent}
total_covidreq = {'global': total_covidreq}
total_covidfund = {'global': total_covidfund}
total_covidpercent = {'global': total_covidpercent}
total_gbvfund = {'global': total_gbvfund}
return [['RequiredFunding', 'Funding', 'PercentFunded',
'RequiredGHRPCovidFunding', 'GHRPCovidFunding', 'GHRPCovidPercentFunded', 'GHRPGBVFunding'], ghxltags], \
[total_allreq, total_allfund, total_allpercent, total_covidreq, total_covidfund, total_covidpercent, total_gbvfund], \
[(hxltag, today_str, 'OCHA', fts_configuration['source_url']) for hxltag in ghxltags], \
[['RequiredHRPFunding', 'HRPFunding', 'HRPPercentFunded',
'RequiredHRPCovidFunding', 'HRPCovidFunding', 'HRPCovidPercentFunded', 'HRPGBVFunding',
'OtherPlans', 'RequiredOtherPlansFunding', 'OtherPlansFunding', 'OtherPlansPercentFunded'],
hxltags], \
[hrp_requirements, hrp_funding, hrp_percentage, hrp_covid_requirements, hrp_covid_funding, hrp_covid_percentage, hrp_gbv_funding,
other_planname, other_requirements, other_funding, other_percentage], \
[(hxltag, today_str, 'OCHA', fts_configuration['source_url']) for hxltag in hxltags]
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
with Download() as downloader:
constants = float_value_convert(
downloader.download_tabular_key_value(
configuration['constants_url']))
constants['Lighting Grid Tier'] = int(constants['Lighting Grid Tier'])
camp_overrides = downloader.download_tabular_cols_as_dicts(
configuration['camp_overrides_url'])
camp_overrides['Population'] = integer_value_convert(
camp_overrides['Population'], dropfailedvalues=True)
camp_overrides['Country'] = key_value_convert(
camp_overrides['Country'], valuefn=get_iso3)
datasets = Dataset.search_in_hdx('displacement',
fq='organization:unhcr')
all_camps_per_country, unhcr_non_camp, unhcr_camp, unhcr_camp_excluded = \
get_camp_non_camp_populations(constants['Non Camp Types'], constants['Camp Types'],
camp_overrides, datasets, downloader)
country_totals = copy.deepcopy(all_camps_per_country)
world_bank_url = configuration['world_bank_url']
urbanratios = get_worldbank_series(
world_bank_url % configuration['urban_ratio_wb'], downloader)
slumratios = get_slumratios(configuration['slum_ratio_url'],
downloader)
noncamp_elec_access = dict()
noncamp_elec_access['Urban'] = get_worldbank_series(
world_bank_url % configuration['urban_elec_wb'], downloader)
noncamp_elec_access['Rural'] = get_worldbank_series(
world_bank_url % configuration['rural_elec_wb'], downloader)
noncamp_elec_access['Slum'] = avg_dicts(noncamp_elec_access['Rural'],
noncamp_elec_access['Urban'])
ieadata = downloader.download_tabular_cols_as_dicts(
configuration['iea_data_url'])
elecappliances = key_value_convert(ieadata['Electrical Appliances'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
cookinglpg = key_value_convert(ieadata['Cooking LPG'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
elecgridtiers = key_value_convert(
downloader.download_tabular_key_value(
configuration['elec_grid_tiers_url']),
keyfn=int,
valuefn=float)
elecgriddirectenergy = float_value_convert(
downloader.download_tabular_key_value(
configuration['elec_grid_direct_energy_url']))
elecgridco2 = key_value_convert(downloader.download_tabular_key_value(
configuration['elec_grid_co2_url']),
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
def get_elecgridco2(iso, inf):
elgridco2 = elecgridco2.get(iso)
if elgridco2 is None:
elgridco2, reg = model.calculate_regional_average(
'Grid CO2', elecgridco2, iso)
inf.append('elco2(%s)=%.3g' % (reg, elgridco2))
return elgridco2
noncamptypes = downloader.download_tabular_cols_as_dicts(
configuration['noncamp_types_url'])
noncamplightingoffgridtypes = integer_value_convert(
noncamptypes['Lighting OffGrid'])
noncampcookingsolidtypes = integer_value_convert(
noncamptypes['Cooking Solid'])
camptypes = get_camptypes(configuration['camp_types_url'], downloader)
camptypes_fallbacks_offgrid, camptypes_fallbacks_solid = \
get_camptypes_fallbacks(configuration['camp_types_fallbacks_url'], downloader, keyfn=get_iso3)
costs = downloader.download_tabular_cols_as_dicts(
configuration['costs_url'])
lightingoffgridcost = float_value_convert(costs['Lighting OffGrid'])
cookingsolidcost = float_value_convert(costs['Cooking Solid'])
noncamp_nonsolid_access = downloader.download_tabular_cols_as_dicts(
configuration['noncamp_cooking_nonsolid_url'])
noncamp_nonsolid_access['Urban'] = key_value_convert(
noncamp_nonsolid_access['Urban'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
noncamp_nonsolid_access['Rural'] = key_value_convert(
noncamp_nonsolid_access['Rural'],
keyfn=get_iso3,
valuefn=float,
dropfailedkeys=True)
noncamp_nonsolid_access['Slum'] = noncamp_nonsolid_access['Urban']
small_camptypes = get_camptypes(configuration['small_camptypes_url'],
downloader)
small_camp_data = downloader.download_tabular_cols_as_dicts(
configuration['small_camps_data_url'])
smallcamps = float_value_convert(small_camp_data['Population'])
small_camps_elecgridco2 = float_value_convert(
small_camp_data['Electricity Grid CO2'])
type_descriptions = downloader.download_tabular_cols_as_dicts(
configuration['type_descriptions_url'])
lighting_type_descriptions = type_descriptions['Lighting Descriptions']
cooking_type_descriptions = type_descriptions['Cooking Descriptions']
model = ChathamHouseModel(constants)
pop_types = ['Urban', 'Slum', 'Rural', 'Camp', 'Small Camp']
headers = list()
results = list()
for i, pop_type in enumerate(pop_types):
results.append(list())
if pop_type == 'Camp':
headers.append(['ISO3 Country Code', 'Country Name', 'Camp Name'])
hxlheaders = ['#country+code', '#country+name', '#loc+name']
elif pop_type == 'Small Camp':
headers.append(['Region'])
hxlheaders = ['#region+name']
else:
headers.append(['ISO3 Country Code', 'Country Name'])
hxlheaders = ['#country+code', '#country+name']
headers[-1].extend(['Population', 'Tier'])
hxlheaders.extend(['#population+num', '#indicator+tier'])
if pop_type not in ['Camp', 'Small Camp']:
headers[-1].extend(
['Grid Expenditure ($m/yr)', 'Grid CO2 Emissions (t/yr)'])
hxlheaders.extend([
'#indicator+value+grid+expenditure',
'#indicator+value+grid+co2_emissions'
])
headers[-1].extend([
'Offgrid Type', 'Lighting Type Description',
'Offgrid Expenditure ($m/yr)', 'Offgrid Capital Costs ($m)',
'Offgrid CO2 Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+type+offgrid', '#indicator+text+lighting',
'#indicator+value+offgrid+expenditure',
'#indicator+value+offgrid+capital_costs',
'#indicator+value+offgrid+co2_emissions'
])
if pop_type not in ['Camp', 'Small Camp']:
headers[-1].extend([
'Nonsolid Expenditure ($m/yr)', 'Nonsolid CO2 Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+value+nonsolid+expenditure',
'#indicator+value+nonsolid+co2_emissions'
])
headers[-1].extend([
'Solid Type', 'Cooking Type Description',
'Solid Expenditure ($m/yr)', 'Solid Capital Costs ($m)',
'Solid CO2_Emissions (t/yr)'
])
hxlheaders.extend([
'#indicator+type+solid', '#indicator+text+cooking',
'#indicator+value+solid+expenditure',
'#indicator+value+solid+capital_costs',
'#indicator+value+solid+co2_emissions'
])
if pop_type != 'Small Camp':
headers[-1].append('Info')
hxlheaders.append('#meta+info')
results[i].append(hxlheaders)
results.append(list())
headers.append(['ISO3 Country Code', 'Country Name', 'Population'])
hxlheaders = ['#country+code', '#country+name', '#population+num']
results[len(results) - 1].append(hxlheaders)
results.append(list())
headers.append([
'ISO3 Country Code', 'Country Name', 'Camp', 'Tier',
'Cooking Spending', 'Cooking Description',
'Population not using Biomass', 'Population using Biomass',
'Lighting Spending', 'Lighting Description', 'Population on Grid',
'Population off Grid'
])
results.append(list())
headers.append([
'code', 'title', 'value', 'latest_date', 'source', 'source_link',
'notes', 'explore', 'units'
])
today = datetime.utcnow()
for iso3 in sorted(unhcr_non_camp):
info = list()
population = model.sum_population(unhcr_non_camp, iso3,
all_camps_per_country)
number_hh_by_pop_type = model.calculate_population(
iso3, population, urbanratios, slumratios, info)
country_elecappliances = elecappliances.get(iso3)
if country_elecappliances is None:
country_elecappliances, region = \
model.calculate_regional_average('Electrical Appliances', elecappliances, iso3)
info.append('elap(%s)=%.3g' % (region, country_elecappliances))
country_elecgridco2 = get_elecgridco2(iso3, info)
country_cookinglpg = cookinglpg.get(iso3)
if country_cookinglpg is None:
country_cookinglpg, region = model.calculate_regional_average(
'LPG', cookinglpg, iso3)
info.append('lpg(%s)=%.3g' % (region, country_elecappliances))
cn = Country.get_country_name_from_iso3(iso3)
for pop_type in number_hh_by_pop_type:
model.reset_pop_counters()
info2 = copy.deepcopy(info)
number_hh = number_hh_by_pop_type[pop_type]
country_elec_access = noncamp_elec_access[pop_type].get(iso3)
if country_elec_access is None:
country_elec_access, region = \
model.calculate_regional_average('Grid access', noncamp_elec_access[pop_type], iso3)
info2.append('elac(%s)=%.3g' %
(region, country_elecappliances))
hh_grid_access, hh_offgrid = model.calculate_hh_access(
number_hh, country_elec_access)
pop_grid_access = model.calculate_population_from_hh(
hh_grid_access)
pop_offgrid_access = model.calculate_population_from_hh(hh_offgrid)
model.pop_grid += pop_grid_access
country_noncamp_nonsolid_access = noncamp_nonsolid_access[
pop_type].get(iso3)
if country_noncamp_nonsolid_access is None:
country_noncamp_nonsolid_access, region = \
model.calculate_regional_average('Nonsolid access', noncamp_nonsolid_access[pop_type], iso3)
info2.append('nsac(%s)=%.3g' %
(region, country_elecappliances))
hh_nonsolid_access, hh_no_nonsolid_access = \
model.calculate_hh_access(number_hh, country_noncamp_nonsolid_access)
pop_biomass_access = model.calculate_population_from_hh(
hh_no_nonsolid_access)
pop_nonbiomass_access = model.calculate_population_from_hh(
hh_nonsolid_access)
model.pop_nonbiomass += pop_nonbiomass_access
ge, gc = model.calculate_ongrid_lighting(hh_grid_access,
elecgridtiers,
country_elecappliances,
country_elecgridco2)
ne, nc = model.calculate_non_solid_cooking(hh_nonsolid_access,
country_cookinglpg)
for tier in model.tiers:
info3 = copy.deepcopy(info2)
noncamplightingoffgridtype = model.get_noncamp_type(
noncamplightingoffgridtypes, pop_type, tier)
noncampcookingsolidtype = model.get_noncamp_type(
noncampcookingsolidtypes, pop_type, tier)
res = model.calculate_offgrid_solid(
tier, hh_offgrid, lighting_type_descriptions,
noncamplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2,
hh_no_nonsolid_access, cooking_type_descriptions,
noncampcookingsolidtype, cookingsolidcost)
noncamplightingtypedesc, oe, oc, oco2, noncampcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures(iso3,
cn,
pop_type,
tier,
se,
oe,
noncampcookingtypedesc,
pop_biomass_access,
noncamplightingtypedesc,
pop_offgrid_access,
results,
ne=ne,
ge=ge)
population = model.calculate_population_from_hh(number_hh)
info3 = ','.join(info3)
row = [
iso3, cn, population, tier, ge, gc,
noncamplightingoffgridtype, noncamplightingtypedesc, oe,
oc, oco2, ne, nc, noncampcookingsolidtype,
noncampcookingtypedesc, se, sc, sco2, info3
]
results[pop_types.index(pop_type.capitalize())].append(row)
camp_offgridtypes_in_countries = dict()
camp_solidtypes_in_countries = dict()
missing_from_unhcr = list()
for name in sorted(camptypes):
model.reset_pop_counters()
info = list()
unhcrcampname = name
result = unhcr_camp.get(unhcrcampname)
if result is None:
firstpart = name.split(':')[0].strip()
for unhcrcampname in sorted(unhcr_camp):
if firstpart in unhcrcampname:
result = unhcr_camp[unhcrcampname]
logger.info(
'Matched first part of name of %s to UNHCR name: %s' %
(name, unhcrcampname))
info.append('Matched %s' % firstpart)
break
if result is None:
camptype = unhcr_camp_excluded.get(name)
if camptype is None:
if check_name_dispersed(name):
logger.info(
'Camp %s from the spreadsheet has been treated as non-camp!'
% name)
else:
missing_from_unhcr.append(name)
else:
logger.info('Camp %s is in UNHCR data but has camp type %s!' %
(name, camptype))
continue
population, iso3, accommodation_type = result
del all_camps_per_country[iso3][accommodation_type][unhcrcampname]
camp_camptypes = camptypes[name]
number_hh = model.calculate_number_hh(population)
country_elecgridco2 = get_elecgridco2(iso3, info)
for tier in model.tiers:
info2 = copy.deepcopy(info)
camplightingoffgridtype = camp_camptypes.get(
'Lighting OffGrid %s' % tier)
if camplightingoffgridtype is None:
logger.warning('No Lighting OffGrid %s for %s in %s' %
(tier, name, cn))
campcookingsolidtype = camp_camptypes.get('Cooking Solid %s' %
tier)
if campcookingsolidtype is None:
logger.warning('No Cooking Solid %s for %s in %s' %
(tier, name, cn))
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
cn = Country.get_country_name_from_iso3(iso3)
model.add_keyfigures(iso3, cn, name, tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population, results)
info2 = ','.join(info2)
row = [
iso3, cn, name, population, tier, camplightingoffgridtype,
camplightingtypedesc, oe, oc, oco2, campcookingsolidtype,
campcookingtypedesc, se, sc, sco2, info2
]
results[pop_types.index('Camp')].append(row)
if camplightingoffgridtype:
append_value(camp_offgridtypes_in_countries, iso3, tier, name,
camplightingoffgridtype)
if campcookingsolidtype:
append_value(camp_solidtypes_in_countries, iso3, tier, name,
campcookingsolidtype)
logger.info(
'The following camps are in the spreadsheet but not in the UNHCR data : %s'
% ', '.join(missing_from_unhcr))
for iso3 in sorted(country_totals):
info = list()
population = model.sum_population(country_totals, iso3)
cn = Country.get_country_name_from_iso3(iso3)
row = [iso3, cn, population]
results[len(results) - 3].append(row)
extra_camp_types = all_camps_per_country[iso3]
country_elecgridco2 = get_elecgridco2(iso3, info)
for accommodation_type in sorted(extra_camp_types):
camps = extra_camp_types[accommodation_type]
for name in sorted(camps):
model.reset_pop_counters()
info2 = copy.deepcopy(info)
population = camps[name]
if population < 20000:
logger.info(
'Ignoring extra camp %s from UNHCR data with population %s (<20000) and accommodation type %s in country %s.'
% (name, population, accommodation_type, cn))
continue
number_hh = model.calculate_number_hh(population)
offgrid_tiers_in_country = camp_offgridtypes_in_countries.get(
iso3)
if offgrid_tiers_in_country is None:
offgrid_tiers_in_country = camptypes_fallbacks_offgrid.get(
iso3)
if not offgrid_tiers_in_country:
logger.warning(
'Missing fallback for country %s, where UNHCR data has extra camp %s with population %s and accommodation type %s'
% (cn, name, population, accommodation_type))
continue
info2.append('UNHCR only')
for tier in offgrid_tiers_in_country:
info3 = copy.deepcopy(info2)
camplightingoffgridtype = offgrid_tiers_in_country[tier]
if isinstance(camplightingoffgridtype, int):
campcookingsolidtype = camptypes_fallbacks_solid[iso3][
tier]
info3.append('Fallback')
else:
camplightingoffgridtype = model.calculate_mostfrequent(
offgrid_tiers_in_country[tier])
campcookingsolidtype = model.calculate_mostfrequent(
camp_solidtypes_in_countries[iso3][tier])
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, country_elecgridco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures(iso3, cn, name, tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population,
results)
info3 = ','.join(info3)
row = [
iso3, cn, name, population, tier,
camplightingoffgridtype, camplightingtypedesc, oe, oc,
oco2, campcookingsolidtype, campcookingtypedesc, se,
sc, sco2, info3
]
results[pop_types.index('Camp')].append(row)
for region in sorted(smallcamps):
model.reset_pop_counters()
info = list()
population = smallcamps[region]
if not population or population == '-':
continue
number_hh = model.calculate_number_hh(population)
region_camptypes = small_camptypes.get(region)
if region_camptypes is None:
logger.info('Missing camp group %s in small camp types!' % region)
continue
elecco2 = small_camps_elecgridco2[region]
if not elecco2 or elecco2 == '-':
info.append('Blank elco2')
elecco2 = 0
for tier in model.tiers:
info2 = copy.deepcopy(info)
camplightingoffgridtype = region_camptypes['Lighting OffGrid %s' %
tier]
campcookingsolidtype = region_camptypes['Cooking Solid %s' % tier]
res = model.calculate_offgrid_solid(
tier, number_hh, lighting_type_descriptions,
camplightingoffgridtype, lightingoffgridcost,
elecgriddirectenergy, elecco2, number_hh,
cooking_type_descriptions, campcookingsolidtype,
cookingsolidcost)
camplightingtypedesc, oe, oc, oco2, campcookingtypedesc, se, sc, sco2 = res
model.add_keyfigures('', region, 'small camp', tier, se, oe,
campcookingtypedesc, population,
camplightingtypedesc, population, results)
info2 = ','.join(info2)
row = [
region,
model.round(population), tier, camplightingoffgridtype,
camplightingtypedesc, oe, oc, oco2, campcookingsolidtype,
campcookingtypedesc, se, sc, sco2, info2
]
results[pop_types.index('Small Camp')].append(row)
date = today.date().isoformat()
source = 'Estimate from the Moving Energy Initiative'
data_url = 'https://data.humdata.org/dataset/energy-consumption-of-refugees-and-displaced-people'
rows = [
[
'MEI01',
'% of Refugees and Displaced People Cooking with Biomass in Camps',
model.get_camp_percentage_biomass(), date, source, data_url, '',
'', 'ratio'
],
[
'MEI02', '% of Refugees and Displaced People Off-Grid in Camps',
model.get_camp_percentage_offgrid(), date, source, data_url, '',
'', 'ratio'
],
[
'MEI03',
'Total Annual Energy Spending by Refugees and Displaced People',
model.get_total_spending(), date, source, data_url, '', '',
'dollars_million'
],
[
'MEI04', 'No. of Countries Hosting Refugees and Displaced People',
len(country_totals), date, source, data_url, '', '', 'count'
]
]
results[len(results) - 1].extend(rows)
dataset, resources, showcase = generate_dataset_resources_and_showcase(
pop_types, today)
folder = gettempdir()
file_to_upload = None
for i, _ in enumerate(results):
resource = resources[i]
file_to_upload = join(folder, resource['name'])
write_list_to_csv(results[i], file_to_upload, headers=headers[i])
resource.set_file_to_upload(file_to_upload)
dataset.add_update_resources(resources)
dataset.update_from_yaml()
# dataset.create_in_hdx()
for resource in dataset.get_resources():
name = resource['name'].lower()
if 'figures' in name and 'disagg' not in name:
logger.info('Updating key figures datastore for %s' % name)