def produce_projects_csv_for_country(output_dir, country):
# first get all appeals for this country (could eliminate this duplicative call, but it's not expensive)
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
appeal_ids = appeals.index
# then get all projects corresponding to those appeals and concatenate into one big frame
list_of_projects = [fts_queries.fetch_projects_json_for_appeal_as_dataframe(appeal_id) for appeal_id in appeal_ids]
list_of_non_empty_projects = filter_out_empty_dataframes(list_of_projects)
projects_frame = pd.concat(list_of_non_empty_projects)
write_dataframe_to_csv(projects_frame, build_csv_path(output_dir, 'projects', country=country))
def populate_organization_level_data(country, organizations=None):
"""
Populate data on funding by organization type
"""
if organizations is None:
organizations = get_organizations_indexed_by_name()
# load appeals, analyze each one
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
funding_dataframes_by_appeal = []
for appeal_id, appeal_row in appeals.iterrows():
# first check if there is any funding at all (otherwise API calls will get upset)
if appeal_row['funding'] == 0:
continue
# query funding by recipient, including "carry over" from previous years
funding_by_recipient = fts_queries.fetch_funding_json_for_appeal_as_dataframe(
appeal_id, grouping='Recipient', alias='organisation')
funding_by_recipient['year'] = appeal_row['year']
funding_dataframes_by_appeal.append(funding_by_recipient)
if funding_dataframes_by_appeal:
# combine the data across appeals
funding_by_recipient_overall = pd.concat(funding_dataframes_by_appeal)
# now roll up by organization type and year
grouped = funding_by_recipient_overall.join(
organizations.type).groupby(['type', 'year'])
funding_by_type_year = grouped.funding.sum()
else:
funding_by_type_year = pd.Series() # just an empty Series
for year in range(YEAR_START, YEAR_END + 1):
ngo_funding = 0.
private_org_funding = 0.
un_agency_funding = 0.
if (ORG_TYPE_NGOS, year) in funding_by_type_year:
ngo_funding = funding_by_type_year[(ORG_TYPE_NGOS, year)]
if (ORG_TYPE_PRIVATE_ORGS, year) in funding_by_type_year:
private_org_funding = funding_by_type_year[(ORG_TYPE_PRIVATE_ORGS,
year)]
if (ORG_TYPE_UN_AGENCIES, year) in funding_by_type_year.index:
un_agency_funding = funding_by_type_year[(ORG_TYPE_UN_AGENCIES,
year)]
add_row_to_values('FY190', country, year, ngo_funding)
add_row_to_values('FY200', country, year, private_org_funding)
add_row_to_values('FY210', country, year, un_agency_funding)
def populate_organization_level_data(country, organizations=None):
"""
Populate data on funding by organization type
"""
if organizations is None:
organizations = get_organizations_indexed_by_name()
# load appeals, analyze each one
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
funding_dataframes_by_appeal = []
for appeal_id, appeal_row in appeals.iterrows():
# first check if there is any funding at all (otherwise API calls will get upset)
if appeal_row['funding'] == 0:
continue
# query funding by recipient, including "carry over" from previous years
funding_by_recipient = fts_queries.fetch_funding_json_for_appeal_as_dataframe(
appeal_id, grouping='Recipient', alias='organisation')
funding_by_recipient['year'] = appeal_row['year']
funding_dataframes_by_appeal.append(funding_by_recipient)
if funding_dataframes_by_appeal:
funding_by_recipient_overall = pd.concat(funding_dataframes_by_appeal)
# now roll up by organization type
funding_by_type = funding_by_recipient_overall.join(organizations.type).groupby(['type', 'year']).funding.sum()
else:
funding_by_type = pd.Series() # just an empty Series
for year in range(YEAR_START, YEAR_END + 1):
ngo_funding = 0.
private_org_funding = 0.
un_agency_funding = 0.
if (ORG_TYPE_NGOS, year) in funding_by_type:
ngo_funding = funding_by_type[(ORG_TYPE_NGOS, year)]
if (ORG_TYPE_PRIVATE_ORGS, year) in funding_by_type:
private_org_funding = funding_by_type[(ORG_TYPE_PRIVATE_ORGS, year)]
if (ORG_TYPE_UN_AGENCIES, year) in funding_by_type.index:
un_agency_funding = funding_by_type[(ORG_TYPE_UN_AGENCIES, year)]
add_row_to_values('FY190', country, year, ngo_funding)
add_row_to_values('FY200', country, year, private_org_funding)
add_row_to_values('FY210', country, year, un_agency_funding)
def populate_appeals_level_data(country):
"""
Populate data based on the "appeals" concept in FTS.
If funding data is not associated with an appeal, it will be excluded.
If there was no appeal, fill in zeros for all items.
This unfortunately conflates "zero" vs "missing" data.
"""
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
if not appeals.empty:
# group/sum all appeals by year, columns are now just the numerical ones:
# - current_requirements, emergency_id, funding, original_requirements, pledges
cross_appeals_by_year = appeals.groupby('year').sum().astype(float)
# do the same with Consolidated Appeals Process (CAP)-only
cap_appeals_by_year = appeals[appeals.type == 'CAP'].groupby(
'year').sum().astype(float)
else:
# just re-use the empty frames
cross_appeals_by_year = appeals
cap_appeals_by_year = appeals
for year in range(YEAR_START, YEAR_END + 1):
original_requirements = 0.
current_requirements = 0.
funding = 0.
if year in cross_appeals_by_year.index:
original_requirements = cross_appeals_by_year[
'original_requirements'][year]
current_requirements = cross_appeals_by_year[
'current_requirements'][year]
funding = cross_appeals_by_year['funding'][year]
add_row_to_values('FY010', country, year, original_requirements)
add_row_to_values('FY020', country, year, current_requirements)
add_row_to_values('FY040', country, year, funding)
cap_requirements = 0.
cap_funding = 0.
if year in cap_appeals_by_year.index:
cap_requirements = cap_appeals_by_year['current_requirements'][
year]
cap_funding = cap_appeals_by_year['funding'][year]
add_row_to_values('FA010', country, year, cap_requirements)
add_row_to_values('FA140', country, year, cap_funding)
def populate_appeals_level_data(country):
"""
Populate data based on the "appeals" concept in FTS.
If funding data is not associated with an appeal, it will be excluded.
If there was no appeal, fill in zeros for all items.
This unfortunately conflates "zero" vs "missing" data.
"""
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
if not appeals.empty:
# group all appeals by year, columns are now just the numerical ones:
# - current_requirements, emergency_id, funding, original_requirements, pledges
cross_appeals_by_year = appeals.groupby('year').sum().astype(float)
# Consolidated Appeals Process (CAP)-only
cap_appeals_by_year = appeals[appeals.type == 'CAP'].groupby('year').sum().astype(float)
else:
# just re-use the empty frames
cross_appeals_by_year = appeals
cap_appeals_by_year = appeals
for year in range(YEAR_START, YEAR_END + 1):
original_requirements = 0.
current_requirements = 0.
funding = 0.
if year in cross_appeals_by_year.index:
original_requirements = cross_appeals_by_year['original_requirements'][year]
current_requirements = cross_appeals_by_year['current_requirements'][year]
funding = cross_appeals_by_year['funding'][year]
add_row_to_values('FY010', country, year, original_requirements)
add_row_to_values('FY020', country, year, current_requirements)
add_row_to_values('FY040', country, year, funding)
cap_requirements = 0.
cap_funding = 0.
if year in cap_appeals_by_year.index:
cap_requirements = cap_appeals_by_year['current_requirements'][year]
cap_funding = cap_appeals_by_year['funding'][year]
add_row_to_values('FA010', country, year, cap_requirements)
add_row_to_values('FA140', country, year, cap_funding)
def populate_organization_level_data(country, organizations=None):
if organizations is None:
organizations = get_organizations_indexed_by_name()
# load appeals, analyze each one
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
funding_dataframes_by_appeal = []
for appeal_id, appeal_row in appeals.iterrows():
# first check if there is any funding at all (otherwise API calls will get upset)
if appeal_row['funding'] == 0:
continue
# query funding by recipient, including "carry over" from previous years
funding_by_recipient = fts_queries.fetch_funding_json_for_appeal_as_dataframe(
appeal_id, grouping='Recipient', alias='organisation')
funding_by_recipient['year'] = appeal_row['year']
funding_dataframes_by_appeal.append(funding_by_recipient)
if not funding_dataframes_by_appeal:
return
funding_by_recipient_overall = pd.concat(funding_dataframes_by_appeal)
# now roll up by organization type
funding_by_type = funding_by_recipient_overall.join(organizations.type).groupby(['type', 'year']).sum()
for (org_type, year), row in funding_by_type.iterrows():
if org_type == ORG_TYPE_NGOS:
add_row_to_values('FY190', country, year, row['funding'])
elif org_type == ORG_TYPE_PRIVATE_ORGS:
add_row_to_values('FY200', country, year, row['funding'])
elif org_type == ORG_TYPE_UN_AGENCIES:
add_row_to_values('FY210', country, year, row['funding'])
else:
# note that some organizations (e.g. Red Cross/Red Crescent) fall outside the 3 indicator categories
print 'Ignoring funding for organization type ' + org_type
def populate_appeals_level_data(country):
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
if appeals.empty:
return
# group all appeals by year, columns are now just the numerical ones:
# - current_requirements, emergency_id, funding, original_requirements, pledges
cross_appeals_by_year = appeals.groupby('year').sum().astype(float)
# iterating by rows is not considered proper form to but it's easier to follow
for year, row in cross_appeals_by_year.iterrows():
add_row_to_values('FY010', country, year, row['original_requirements'])
add_row_to_values('FY020', country, year, row['current_requirements'])
add_row_to_values('FY040', country, year, row['funding'])
# note this is a fraction, not a percent, and not clipped to 100%
add_row_to_values('FY030', country, year, row['funding'] / row['current_requirements'])
# CAP-only
cap_appeals = appeals[appeals.type == 'CAP']
for appeal_id, row in cap_appeals.iterrows():
add_row_to_values('FA010', country, row['year'], row['current_requirements'])
add_row_to_values('FA140', country, row['year'], row['funding'])
def produce_appeals_csv_for_country(output_dir, country):
appeals = fts_queries.fetch_appeals_json_for_country_as_dataframe(country)
write_dataframe_to_csv(appeals, build_csv_path(output_dir, 'appeals', country=country))
