def get_census_profile(geo_code, geo_level, profile_name=None):
logger.info('Begin of transaction for {}: {}'.format(geo_level, geo_code))
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session),
level)
return data
finally:
logger.info('End of transaction for {}: {}'.format(
geo_level, geo_code))
session.close()
def get_census_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session),
level)
# tweaks to make the data nicer
# show X largest groups on their own and group the rest as 'Other'
#group_remainder(data['households']['roofing_material_distribution'], 5)
#group_remainder(data['households']['wall_material_distribution'], 5)
return data
finally:
session.close()
def get_timeseries_profile(geo, profile_name, request):
session = get_session()
try:
comparative_geos = geo_data.get_comparative_geos(geo)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(
geo, session,
request) # calling get_PROFILE_SECTIONS_profile
# get profiles for province and/or country
for comp_geo in comparative_geos:
try:
merge_dicts(data[section],
func(comp_geo, session, request),
comp_geo.geo_level)
except KeyError as e:
msg = "Error merging data into %s for section '%s' from %s: KeyError: %s" % (
geo.geoid, section, comp_geo.geoid, e)
log.fatal(msg, exc_info=e)
raise ValueError(msg)
finally:
session.close()
return data
def get_profile(geo, profile_name, request):
session = get_session()
try:
comp_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
if geo.geo_level not in ['country', 'province', 'municipality']:
pass
# Raise error as we don't have this data
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session)
# get profiles for province and/or country
for comp_geo in comp_geos:
# merge summary profile into current geo profile
merge_dicts(data[section], func(comp_geo, session), comp_geo.geo_level)
group_remainder(data['households']['type_of_dwelling_distribution'], 5)
group_remainder(data['service_delivery']['water_source_distribution'], 5)
group_remainder(data['service_delivery']['toilet_facilities_distribution'], 5)
return data
finally:
session.close()
def get_profile(geo, profile_name, request):
session = get_session()
try:
comp_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
if geo.geo_level not in ['country', 'province', 'municipality']:
pass
# Raise error as we don't have this data
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session)
# get profiles for province and/or country
for comp_geo in comp_geos:
# merge summary profile into current geo profile
merge_dicts(data[section], func(comp_geo, session),
comp_geo.geo_level)
group_remainder(data['households']['type_of_dwelling_distribution'], 5)
group_remainder(data['service_delivery']['water_source_distribution'],
5)
group_remainder(
data['service_delivery']['toilet_facilities_distribution'], 5)
return data
finally:
session.close()
def get_elections_profile(geo_code, geo_level):
data = OrderedDict()
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
for election in ELECTIONS:
section = election['name'].lower().replace(' ', '_')
data[section] = get_election_data(geo_code, geo_level, election, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], get_election_data(code, level, election, session), level)
# tweaks to make the data nicer
# show 8 largest parties on their own and group the rest as 'Other'
group_remainder(data[section]['party_distribution'], 9)
if geo_level == 'country':
add_elections_media_coverage(data)
return data
finally:
session.close()
def get_census_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session),
level)
finally:
session.close()
if geo_level != 'vdc':
group_remainder(data['demographics']['language_distribution'], 10)
group_remainder(data['demographics']['ethnic_distribution'], 10)
return data
def get_census_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session), level)
# tweaks to make the data nicer
# show X largest groups on their own and group the rest as 'Other'
group_remainder(data['households']['roofing_material_distribution'], 5)
group_remainder(data['households']['wall_material_distribution'], 5)
return data
finally:
session.close()
def compare_geos(self):
"""
Get the values for the comparative geo and add it to the main geo
"Note": for land cover we dont have provincial or country, need to skip these for
comparative geo.
"""
comparative_geos = geo_data.get_comparative_geos(self.geo)
for comp_geo in comparative_geos:
try:
if self.profile.dataset_context:
if comp_geo.geo_level not in ("country", "province"):
merge_dicts(
self.distribution,
self.context_comparative_geo(comp_geo),
comp_geo.geo_level,
)
else:
merge_dicts(
self.distribution,
self.comparative_geo(comp_geo),
comp_geo.geo_level,
)
except KeyError as error:
log.warn("Unable to merge dicts for profile %s", self.profile.title)
log.error("Unable to merge dicts: %s", error)
pass
def get_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
sections = list(PROFILE_SECTIONS)
if geo_level not in ['country', 'province', 'district', 'municipality']:
pass
# Raise error as we don't have this data
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session), level)
return data
finally:
session.close()
def get_profile(geo, profile_name, request):
session = get_session()
try:
comp_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
if geo.geo_level not in [
'country', 'province', 'district', 'municipality'
]:
pass
# Raise error as we don't have this data
"""
The following is temporary and enables us to determine what to display for geos:
Within WC: All indicators, with WC as root comparisson geo
Outside WC: Some indicators, with ZA as root comparrison geo
This is beacause Wazimap expects data for all geos.
This will be removed once we have imported all the data.
"""
# There are datasets with only WC information
display_profile = 'WC' if (
geo.geo_code == 'WC' or 'WC' in [cg.geo_code
for cg in comp_geos]) else 'ZA'
data['display_profile'] = display_profile
data['primary_release_year'] = current_context().get('year')
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session, display_profile)
# get profiles for province and/or country
for comp_geo in comp_geos:
# merge summary profile into current geo profile
merge_dicts(
data[section],
func(comp_geo,
session,
display_profile,
comparative=True), comp_geo.geo_level)
# Make data look nicer on profile page
group_remainder(data['demographics']['youth_population_by_language'],
11)
return data
finally:
session.close()
def get_profile(geo, profile_name, request):
session = get_session()
try:
comp_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
if geo.geo_level not in ['country', 'province', 'district', 'municipality']:
pass
# Raise error as we don't have this data
"""
The following is temporary and enables us to determine what to display for geos:
Within WC: All indicators, with WC as root comparisson geo
Outside WC: Some indicators, with ZA as root comparrison geo
This is beacause Wazimap expects data for all geos.
This will be removed once we have imported all the data.
"""
# There are datasets with only WC information
display_profile = 'WC' if (geo.geo_code == 'WC' or 'WC' in [cg.geo_code for cg in comp_geos]) else 'ZA'
data['display_profile'] = display_profile
data['primary_release_year'] = current_context().get('year')
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session, display_profile)
# get profiles for province and/or country
for comp_geo in comp_geos:
# merge summary profile into current geo profile
merge_dicts(data[section], func(comp_geo, session, display_profile, comparative=True), comp_geo.geo_level)
# Make data look nicer on profile page
group_remainder(data['demographics']['youth_population_by_language'], 11)
return data
finally:
session.close()
def get_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
for level, code in geo_summary_levels:
merge_dicts(data[section], func(code, level, session), level)
return data
finally:
session.close()
def get_profile(geo, profile_name, request):
session = get_session()
try:
comparative_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = []
for cat in SECTIONS:
sections.extend(SECTIONS[cat]['profiles'])
for section in sections:
section = section.lower().replace(' ', '_')
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
# get profiles for comperative geometries
data[section] = func(geo, session)
# get profiles for comparative geometries
for comp_geo in comparative_geos:
try:
merge_dicts(data[section], func(comp_geo, session),
comp_geo.geo_level)
except KeyError as e:
msg = "Error merging data into %s for section '%s' from %s: KeyError: %s" % (
geo.geoid, section, comp_geo.geoid, e)
log.fatal(msg, exc_info=e)
raise ValueError(msg)
# tweaks to make the data nicer
# show X largest groups on their own and group the rest as 'Other'
if 'households' in sections:
group_remainder(
data['households']['roofing_material_distribution'], 5)
group_remainder(data['households']['wall_material_distribution'],
5)
return data
finally:
session.close()
def get_profile(geo, profile_name, request):
session = get_session()
try:
comparative_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
if geo.geo_level in ['country', 'province']:
sections.append('crime')
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session)
# get profiles for comparative geometries
for comp_geo in comparative_geos:
try:
merge_dicts(data[section], func(comp_geo, session), comp_geo.geo_level)
except KeyError as e:
msg = "Error merging data into %s for section '%s' from %s: KeyError: %s" % (geo.geoid, section, comp_geo.geoid, e)
log.fatal(msg, exc_info=e)
raise ValueError(msg)
finally:
session.close()
# tweaks to make the data nicer
# show 3 largest groups on their own and group the rest as 'Other'
group_remainder(data['service_delivery']['water_source_distribution'], 5)
group_remainder(data['service_delivery']['refuse_disposal_distribution'], 5)
group_remainder(data['service_delivery']['toilet_facilities_distribution'], 5)
group_remainder(data['demographics']['language_distribution'], 7)
group_remainder(data['demographics']['province_of_birth_distribution'], 7)
group_remainder(data['demographics']['region_of_birth_distribution'], 5)
group_remainder(data['households']['type_of_dwelling_distribution'], 5)
group_remainder(data['child_households']['type_of_dwelling_distribution'], 5)
data['elections'] = get_elections_profile(geo)
return data
def get_profile(geo_code, geo_level, profile_name=None):
session = get_session()
try:
geo_summary_levels = geo_data.get_summary_geo_info(geo_code, geo_level)
data = {}
sections = list(PROFILE_SECTIONS)
if geo_level in ['country', 'province']:
sections.append('crime')
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session), level)
finally:
session.close()
# tweaks to make the data nicer
# show 3 largest groups on their own and group the rest as 'Other'
group_remainder(data['service_delivery']['water_source_distribution'], 5)
group_remainder(data['service_delivery']['refuse_disposal_distribution'], 5)
group_remainder(data['service_delivery']['toilet_facilities_distribution'], 5)
group_remainder(data['demographics']['language_distribution'], 7)
group_remainder(data['demographics']['province_of_birth_distribution'], 7)
group_remainder(data['demographics']['region_of_birth_distribution'], 5)
group_remainder(data['households']['type_of_dwelling_distribution'], 5)
group_remainder(data['child_households']['type_of_dwelling_distribution'], 5)
data['elections'] = get_elections_profile(geo_code, geo_level)
return data
def get_land_profile(geo, profile_name, request):
session = get_session()
try:
comparative_geos = geo_data.get_comparative_geos(geo)
data = {}
sections = list(PROFILE_SECTIONS)
for section in sections:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo, session)
# get profiles for comparative geometries
if not data[section]['is_missing']:
for comp_geo in comparative_geos:
try:
merge_dicts(
data[section], func(
comp_geo, session), comp_geo.geo_level)
except KeyError as e:
msg = "Error merging data into %s for section '%s' from %s: KeyError: %s" % (
geo.geoid, section, comp_geo.geoid, e)
log.fatal(msg, exc_info=e)
raise ValueError(msg)
data['land_audit'] = get_land_audit_profile(geo, session)
if geo.geo_level in ['district', 'municipality']:
data['districtdistribution'] = get_districtdistribution_profile(geo, session)
return data
finally:
session.close()
def get_elections_profile(geo):
ELECTIONS = [
{
'name': 'Municipal 2016',
'table_code': 'municipal_2016',
'dataset': '2016 Municipal Elections',
'geo_version': '2016',
'release_year': '2016'
},
]
if geo.version == '2011' or geo.geo_level != 'ward':
ELECTIONS.extend([
{
'name': 'National 2014',
'table_code': 'national_2014',
'dataset': '2014 National Elections',
'geo_version': '2011',
'release_year': '2014'
},
{
'name': 'Provincial 2014',
'table_code': 'provincial_2014',
'dataset': '2014 Provincial Elections',
'geo_version': '2011',
'release_year': '2014'
},
{
'name': 'Municipal 2011',
'table_code': 'municipal_2011',
'dataset': '2011 Municipal Elections',
'geo_version': '2011',
'release_year': '2011'
},
])
data = OrderedDict()
session = get_session()
try:
comparative_geos = geo_data.get_comparative_geos(geo)
for election in ELECTIONS:
section = election['name'].lower().replace(' ', '_')
# TODO: Hack to request data for different geo_version than this geo.
actual_geo_version = geo.version
geo.version = election['geo_version']
# If we can't find election data with the relevant geo version then
# we don't want to show anything for this election.
election_data = None
try:
election_data = get_election_data(geo, election, session)
except DataNotFound:
pass
finally:
geo.version = actual_geo_version
if election_data:
data[section] = election_data
# get profiles for province and/or country
for comp_geo in comparative_geos:
comp_geo.version = election['geo_version']
merge_dicts(data[section], get_election_data(comp_geo, election, session), comp_geo.geo_level)
# tweaks to make the data nicer
# show 8 largest parties on their own and group the rest as 'Other'
group_remainder(data[section]['party_distribution'], 9)
if geo.geo_level == 'country':
add_elections_media_coverage(data)
return data
finally:
session.close()
