def get_map_points():
"""
Get map point data for the given place type enclosed within the given dcid
"""
place_dcid = request.args.get("placeDcid")
if not place_dcid:
return Response(json.dumps("error: must provide a placeDcid field"),
400,
mimetype='application/json')
place_type = request.args.get("placeType")
if not place_type:
return Response(json.dumps("error: must provide a placeType field"),
400,
mimetype='application/json')
geos = []
geos = dc_service.get_places_in([place_dcid],
place_type).get(place_dcid, [])
if not geos:
return Response(json.dumps({}), 200, mimetype='application/json')
names_by_geo = place_api.get_display_name('^'.join(geos), g.locale)
# For some places, lat long is attached to the place node, but for other
# places, the lat long is attached to the location value of the place node.
# If a place has location, we will use the location value to find the lat
# and long.
# eg. epaGhgrpFacilityId/1003010 has latitude and longitude but no location
# epa/120814013 which is an AirQualitySite has a location, but no latitude
# or longitude
location_by_geo = dc_service.get_property_values(geos, "location")
# dict of <dcid used to get latlon>: <dcid of the place>
geo_by_latlon_subject = {}
for geo_dcid in geos:
if geo_dcid in location_by_geo and len(
location_by_geo.get(geo_dcid)) > 0:
location_dcid = location_by_geo[geo_dcid][0]
geo_by_latlon_subject[location_dcid] = geo_dcid
else:
geo_by_latlon_subject[geo_dcid] = geo_dcid
lat_by_subject = dc_service.get_property_values(
list(geo_by_latlon_subject.keys()), "latitude")
lon_by_subject = dc_service.get_property_values(
list(geo_by_latlon_subject.keys()), "longitude")
map_points_list = []
for subject_dcid, latitude in lat_by_subject.items():
longitude = lon_by_subject.get(subject_dcid, [])
if len(latitude) == 0 or len(longitude) == 0:
continue
if not is_float(latitude[0]) or not is_float(longitude[0]):
continue
geo_id = geo_by_latlon_subject.get(subject_dcid, "")
map_point = {
"placeDcid": geo_id,
"placeName": names_by_geo.get(geo_id, "Unnamed Place"),
"latitude": float(latitude[0]),
"longitude": float(longitude[0])
}
map_points_list.append(map_point)
return Response(json.dumps(map_points_list),
200,
mimetype='application/json')
def ranking_api(stat_var, place_type, place=None):
"""Returns top 100, bottom 100 or all rankings for a stats var, grouped
by place type and optionally scoped by a containing place (all is
returned if there are fewer than 100 places in the group). Each place in
the ranking has it's name returned, if available.
Optional GET args:
pc (per capita - the presence of the key enables it)
bottom (show bottom ranking instead - the presence of the key enables it)
"""
is_per_capita = flask.request.args.get('pc', False) != False
is_show_bottom = flask.request.args.get('bottom', False) != False
rank_keys = BOTTOM_KEYS_KEEP if is_show_bottom else TOP_KEYS_KEEP
delete_keys = BOTTOM_KEYS_DEL if is_show_bottom else TOP_KEYS_DEL
ranking_results = dc.get_place_ranking([stat_var], place_type, place,
is_per_capita)
if not 'payload' in ranking_results:
flask.abort(500)
payload = ranking_results['payload']
if not stat_var in ranking_results['payload']:
flask.abort(500)
# split rankAll to top/bottom if it's larger than RANK_SIZE
if 'rankAll' in payload[stat_var]:
rank_all = payload[stat_var]['rankAll']
if 'info' in rank_all and len(rank_all['info']) > RANK_SIZE:
if is_show_bottom:
payload[stat_var]['rankBottom1000'] = rank_all
else:
payload[stat_var]['rankTop1000'] = rank_all
del payload[stat_var]['rankAll']
for k in delete_keys:
try:
del payload[stat_var][k]
except KeyError:
pass # key might not exist for fewer than 1000 places
dcids = set()
for k in rank_keys:
if k in payload[stat_var] and 'info' in payload[stat_var][k]:
info = payload[stat_var][k]['info']
if is_show_bottom:
# truncate and reverse the bottom RANK_SIZE elems
info = info[-RANK_SIZE:]
else:
info = info[:RANK_SIZE]
for r in info:
dcids.add(r['placeDcid'])
payload[stat_var][k]['info'] = info
# payload[stat_var][k]['count'] = count
place_names = place_api.get_display_name('^'.join(sorted(list(dcids))))
for k in rank_keys:
if k in payload[stat_var] and 'info' in payload[stat_var][k]:
for r in payload[stat_var][k]['info']:
r['placeName'] = place_names[r['placeDcid']]
return flask.Response(json.dumps(ranking_results),
200,
mimetype='application/json')
def geojson():
"""
Get geoJson data for places enclosed within the given dcid
"""
place_dcid = request.args.get("placeDcid")
if not place_dcid:
return Response(json.dumps("error: must provide a placeDcid field"),
400,
mimetype='application/json')
place_type = request.args.get("placeType")
if not place_type:
place_dcid, place_type = get_choropleth_display_level(place_dcid)
geos = []
if place_dcid and place_type:
geos = dc_service.get_places_in([place_dcid],
place_type).get(place_dcid, [])
if not geos:
return Response(json.dumps({}), 200, mimetype='application/json')
geojson_prop = CHOROPLETH_GEOJSON_PROPERTY_MAP.get(place_type, "")
# geoId/72 needs higher resolution geojson because otherwise, the map looks
# too fragmented
if place_dcid == 'geoId/72':
geojson_prop = 'geoJsonCoordinatesDP1'
names_by_geo = place_api.get_display_name('^'.join(geos), g.locale)
geojson_by_geo = dc_service.get_property_values(geos, geojson_prop)
features = []
for geo_id, json_text in geojson_by_geo.items():
if json_text and geo_id in names_by_geo:
geo_feature = {
"type": "Feature",
"geometry": {
"type": "MultiPolygon",
},
"id": geo_id,
"properties": {
"name": names_by_geo.get(geo_id, "Unnamed Area"),
"geoDcid": geo_id,
}
}
# Load, simplify, and add geoJSON coordinates.
# Exclude geo if no or multiple renderings are present.
if len(json_text) != 1:
continue
geojson = json.loads(json_text[0])
geo_feature['geometry']['coordinates'] = (
coerce_geojson_to_righthand_rule(geojson['coordinates'],
geojson['type']))
features.append(geo_feature)
return Response(json.dumps({
"type": "FeatureCollection",
"features": features,
"properties": {
"current_geo": place_dcid
}
}),
200,
mimetype='application/json')
def homepage():
place_names = place_api.get_display_name('^'.join(_HOMEPAGE_PLACE_DCIDS),
g.locale)
blog_date = babel_dates.format_date(date(2021, 6, 1),
format='long',
locale=g.locale)
return render_template('static/homepage.html',
place_names=place_names,
blog_date=blog_date)
def homepage():
if current_app.config['PRIVATE']:
return render_template('static/private.html')
place_names = place_api.get_display_name('^'.join(_HOMEPAGE_PLACE_DCIDS),
g.locale)
blog_date = babel_dates.format_date(date(2021, 7, 26),
format='long',
locale=g.locale)
return render_template('static/homepage.html',
place_names=place_names,
blog_date=blog_date)
def place(place_dcid=None):
redirect_args = dict(flask.request.args)
should_redirect = False
if 'topic' in flask.request.args:
redirect_args['category'] = flask.request.args.get('topic', '')
del redirect_args['topic']
should_redirect = True
category = redirect_args.get('category', None)
if category in CATEGORY_REDIRECTS:
redirect_args['category'] = CATEGORY_REDIRECTS[category]
should_redirect = True
if should_redirect:
redirect_args['place_dcid'] = place_dcid
return flask.redirect(flask.url_for('place.place', **redirect_args))
dcid = flask.request.args.get('dcid', None)
if dcid:
# Traffic from "explore more" in Search. Forward along all parameters,
# except for dcid, to the new URL format.
redirect_args = dict(flask.request.args)
redirect_args['place_dcid'] = dcid
del redirect_args['dcid']
redirect_args['category'] = category
url = flask.url_for('place.place',
**redirect_args,
_external=True,
_scheme=current_app.config.get('SCHEME', 'https'))
return flask.redirect(url)
if not place_dcid:
# Use display names (including state, if applicable) for the static page
place_names = place_api.get_display_name(
'^'.join(_PLACE_LANDING_DCIDS), g.locale)
return flask.render_template(
'place_landing.html',
place_names=place_names,
maps_api_key=current_app.config['MAPS_API_KEY'])
place_type = place_api.get_place_type(place_dcid)
place_names = place_api.get_i18n_name([place_dcid])
if place_names and place_names.get(place_dcid):
place_name = place_names[place_dcid]
else:
place_name = place_dcid
return flask.render_template(
'place.html',
place_type=place_type,
place_name=place_name,
place_dcid=place_dcid,
category=category if category else '',
maps_api_key=current_app.config['MAPS_API_KEY'])
def geojson(dcid):
"""
Get geoJson data for a given place
"""
geos, geojson_prop = get_choropleth_places(dcid)
if not geos:
return Response(json.dumps({}), 200, mimetype='application/json')
names_by_geo = place_api.get_display_name('^'.join(geos), g.locale)
geojson_by_geo = dc_service.get_property_values(geos, geojson_prop)
features = []
for geo_id, json_text in geojson_by_geo.items():
if json_text and geo_id in names_by_geo:
geo_feature = {
"type": "Feature",
"geometry": {
"type": "MultiPolygon",
},
"id": geo_id,
"properties": {
"name": names_by_geo.get(geo_id, "Unnamed Area"),
"hasSublevel": False,
"geoDcid": geo_id,
}
}
# Load, simplify, and add geoJSON coordinates.
# Exclude geo if no or multiple renderings are present.
if len(json_text) != 1:
continue
geojson = json.loads(json_text[0])
geo_feature['geometry']['coordinates'] = (
choropleth_api.coerce_geojson_to_righthand_rule(
geojson['coordinates'], geojson['type']))
features.append(geo_feature)
return Response(json.dumps({
"type": "FeatureCollection",
"features": features,
"properties": {
"current_geo": dcid
}
}),
200,
mimetype='application/json')
def place(place_dcid=None):
dcid = flask.request.args.get('dcid', None)
topic = flask.request.args.get('topic', None)
if dcid:
# Traffic from "explore more" in Search. Forward along all parameters,
# except for dcid, to the new URL format.
redirect_args = dict(flask.request.args)
redirect_args['place_dcid'] = dcid
del redirect_args['dcid']
redirect_args['topic'] = topic
url = flask.url_for('place.place',
**redirect_args,
_external=True,
_scheme=current_app.config.get('SCHEME', 'https'))
return flask.redirect(url)
if not place_dcid:
# Use display names (including state, if applicable) for the static page
place_names = place_api.get_display_name(
'^'.join(_PLACE_LANDING_DCIDS), g.locale)
return flask.render_template(
'place_landing.html',
place_names=place_names,
maps_api_key=current_app.config['MAPS_API_KEY'])
place_type = place_api.get_place_type(place_dcid)
place_names = place_api.get_i18n_name([place_dcid])
if place_names:
place_name = place_names[place_dcid]
else:
place_name = place_dcid
return flask.render_template(
'place.html',
place_type=place_type,
place_name=place_name,
place_dcid=place_dcid,
topic=topic if topic else '',
maps_api_key=current_app.config['MAPS_API_KEY'])
def get_map_points():
"""
Get map point data for the given place type enclosed within the given dcid
"""
place_dcid = request.args.get("placeDcid")
if not place_dcid:
return Response(json.dumps("error: must provide a placeDcid field"),
400,
mimetype='application/json')
place_type = request.args.get("placeType")
if not place_type:
return Response(json.dumps("error: must provide a placeType field"),
400,
mimetype='application/json')
geos = []
geos = dc_service.get_places_in([place_dcid],
place_type).get(place_dcid, [])
if not geos:
return Response(json.dumps({}), 200, mimetype='application/json')
names_by_geo = place_api.get_display_name('^'.join(geos), g.locale)
latitude_by_geo = dc_service.get_property_values(geos, "latitude")
longitude_by_geo = dc_service.get_property_values(geos, "longitude")
map_points_list = []
for geo_id, latitude in latitude_by_geo.items():
longitude = longitude_by_geo.get(geo_id, [])
if len(latitude_by_geo) == 0 or len(longitude) == 0:
continue
map_point = {
"placeDcid": geo_id,
"placeName": names_by_geo.get(geo_id, "Unnamed Place"),
"latitude": float(latitude[0]),
"longitude": float(longitude[0])
}
map_points_list.append(map_point)
return Response(json.dumps(map_points_list),
200,
mimetype='application/json')
def data(dcid):
"""
Get chart spec and stats data of the landing page for a given place.
"""
logging.info("Landing Page: cache miss for %s, fetch and process data ...",
dcid)
target_category = request.args.get("category")
spec_and_stat = build_spec(current_app.config['CHART_CONFIG'])
new_stat_vars = current_app.config['NEW_STAT_VARS']
raw_page_data = get_landing_page_data(dcid, new_stat_vars)
if 'statVarSeries' not in raw_page_data:
logging.info("Landing Page: No data for %s", dcid)
return Response(json.dumps({}), 200, mimetype='application/json')
# Filter out Metropolitan France parent place.
parent_places = [
el for el in raw_page_data.get('parentPlaces', [])
if el != 'country/FXX'
]
raw_page_data['parentPlaces'] = parent_places
# Only US places have comparison charts.
is_usa_place = False
for place in [dcid] + raw_page_data.get('parentPlaces', []):
if place == 'country/USA':
is_usa_place = True
break
all_stat = raw_page_data['statVarSeries']
# Remove empty category and topics
for category in list(spec_and_stat.keys()):
for topic in list(spec_and_stat[category].keys()):
filtered_charts = []
for chart in spec_and_stat[category][topic]:
keep_chart = False
for stat_var in chart['statsVars']:
if all_stat[dcid].get('data', {}).get(stat_var, {}):
keep_chart = True
break
if keep_chart:
filtered_charts.append(chart)
if not filtered_charts:
del spec_and_stat[category][topic]
else:
spec_and_stat[category][topic] = filtered_charts
if not spec_and_stat[category]:
del spec_and_stat[category]
# Populate the data for each chart.
# This is heavy lifting, takes time to process.
for category in spec_and_stat:
# Only process the target category.
if category != target_category:
continue
if category == OVERVIEW:
if is_usa_place:
chart_types = ['nearby', 'child']
else:
chart_types = ['similar']
else:
chart_types = BAR_CHART_TYPES
for topic in spec_and_stat[category]:
for chart in spec_and_stat[category][topic]:
# Trend data
chart['trend'] = get_trend(chart, all_stat, dcid)
if 'aggregate' in chart:
aggregated_stat_vars = list(chart['trend'].get(
'series', {}).keys())
if aggregated_stat_vars:
chart['trend']['statsVars'] = aggregated_stat_vars
else:
chart['trend'] = {}
# Bar data
for t in chart_types:
chart[t] = get_bar(chart, all_stat, [dcid] +
raw_page_data.get(t + 'Places', []))
if t == 'similar' and 'data' in chart[t]:
# If no data for current place, do not serve similar
# place data.
keep_chart = False
for d in chart[t]['data']:
if d['dcid'] == dcid:
keep_chart = True
break
if not keep_chart:
chart[t] = {}
# Update stat vars for aggregated stats
if 'aggregate' in chart and chart[t]:
chart[t]['statsVars'] = []
for place_data in chart[t].get('data', []):
stat_vars = list(place_data['data'].keys())
if len(stat_vars) > len(chart[t]['statsVars']):
chart[t]['statsVars'] = stat_vars
elif len(stat_vars) == 0:
chart[t] = {}
if 'aggregate' in chart:
chart['statsVars'] = []
# Get chart category name translations
categories = {}
for category in list(spec_and_stat.keys()) + list(spec_and_stat[OVERVIEW]):
categories[category] = gettext(f'CHART_TITLE-CHART_CATEGORY-{category}')
# Get display name for all places
all_places = [dcid]
for t in BAR_CHART_TYPES:
all_places.extend(raw_page_data.get(t + 'Places', []))
names = place_api.get_display_name('^'.join(sorted(all_places)), g.locale)
# Pick data to highlight - only population for now
highlight = {}
pop_data = raw_page_data.get('latestPopulation', {}).get(dcid, {})
if pop_data:
population = {
'date': pop_data['date'],
'data': [{
'dcid': dcid,
'data': {
'Count_Person': pop_data['value']
}
}],
'sources': [pop_data['metadata']['provenanceUrl']]
}
highlight = {gettext('CHART_TITLE-Population'): population}
response = {
'pageChart': spec_and_stat,
'allChildPlaces': get_i18n_all_child_places(raw_page_data),
'childPlacesType': raw_page_data.get('childPlacesType', ""),
'childPlaces': raw_page_data.get('childPlaces', []),
'parentPlaces': raw_page_data.get('parentPlaces', []),
'similarPlaces': raw_page_data.get('similarPlaces', []),
'nearbyPlaces': raw_page_data.get('nearbyPlaces', []),
'categories': categories,
'names': names,
'highlight': highlight,
}
return Response(json.dumps(response), 200, mimetype='application/json')
def data(dcid):
"""
Get chart spec and stats data of the landing page for a given place.
"""
logging.info("Landing Page: cache miss for %s, fetch and process data ...",
dcid)
spec_and_stat = build_spec(current_app.config['CHART_CONFIG'])
new_stat_vars = current_app.config['NEW_STAT_VARS']
raw_page_data = get_landing_page_data(dcid, new_stat_vars)
if not 'statVarSeries' in raw_page_data:
logging.info("Landing Page: No data for %s", dcid)
return Response(json.dumps({}), 200, mimetype='application/json')
# Filter out Metropolitan France parent place.
parent_places = [
el for el in raw_page_data.get('parentPlaces', [])
if el != 'country/FXX'
]
raw_page_data['parentPlaces'] = parent_places
# Only US places have comparison charts.
is_usa_place = False
for place in [dcid] + raw_page_data.get('parentPlaces', []):
if place == 'country/USA':
is_usa_place = True
break
# Populate the data for each chart
all_stat = raw_page_data['statVarSeries']
for category in spec_and_stat:
if category == OVERVIEW:
if is_usa_place:
chart_types = ['nearby', 'child']
else:
chart_types = ['similar']
else:
chart_types = BAR_CHART_TYPES
for topic in spec_and_stat[category]:
for chart in spec_and_stat[category][topic]:
# Trend data
chart['trend'] = get_trend(chart, all_stat, dcid)
if 'aggregate' in chart:
aggregated_stat_vars = list(chart['trend'].get(
'series', {}).keys())
if aggregated_stat_vars:
chart['trend']['statsVars'] = aggregated_stat_vars
else:
chart['trend'] = {}
# Bar data
for t in chart_types:
chart[t] = get_bar(chart, all_stat, [dcid] +
raw_page_data.get(t + 'Places', []))
if t == 'similar' and 'data' in chart[t]:
# If no data for current place, do not serve similar
# place data.
keep_chart = False
for d in chart[t]['data']:
if d['dcid'] == dcid:
keep_chart = True
break
if not keep_chart:
chart[t] = {}
# Update stat vars for aggregated stats
if 'aggregate' in chart and chart[t]:
chart[t]['statsVars'] = []
for place_data in chart[t].get('data', []):
stat_vars = list(place_data['data'].keys())
if len(stat_vars) > len(chart[t]['statsVars']):
chart[t]['statsVars'] = stat_vars
elif len(stat_vars) == 0:
chart[t] = {}
if 'aggregate' in chart:
chart['statsVars'] = []
# Remove empty category and topics
for category in list(spec_and_stat.keys()):
for topic in list(spec_and_stat[category].keys()):
filtered_charts = []
for chart in spec_and_stat[category][topic]:
keep_chart = False
for t in ['trend'] + BAR_CHART_TYPES:
if chart.get(t, None):
keep_chart = True
break
if keep_chart:
filtered_charts.append(chart)
if not filtered_charts:
del spec_and_stat[category][topic]
else:
spec_and_stat[category][topic] = filtered_charts
if not spec_and_stat[category]:
del spec_and_stat[category]
# Only keep the "Overview" category if the number of total chart is less
# than certain threshold.
overview_set = set()
non_overview_set = set()
chart_count = 0
# Get the overview charts
for topic, charts in spec_and_stat[OVERVIEW].items():
for chart in charts:
overview_set.add((topic, chart['title']))
chart_count += 1
# Get the non overview charts
for category, topic_data in spec_and_stat.items():
if category == OVERVIEW:
continue
for topic in topic_data:
if (category, topic) not in overview_set:
non_overview_set.add((category, topic))
chart_count += 1
# If the total number of chart is too small, then merge all charts to
# the overview category and remove other categories
if chart_count < MIN_CHART_TO_KEEP_TOPICS:
for category, topic in non_overview_set:
spec_and_stat[OVERVIEW][category].extend(
spec_and_stat[category][topic])
for category in list(spec_and_stat.keys()):
if category != OVERVIEW:
del spec_and_stat[category]
# Get chart category name translations
categories = {}
for category in list(spec_and_stat.keys()) + list(spec_and_stat[OVERVIEW]):
categories[category] = gettext(
f'CHART_TITLE-CHART_CATEGORY-{category}')
# Get display name for all places
all_places = [dcid]
for t in BAR_CHART_TYPES:
all_places.extend(raw_page_data.get(t + 'Places', []))
names = place_api.get_display_name('^'.join(sorted(all_places)), g.locale)
# Pick data to highlight - only population for now
pop_data = raw_page_data['latestPopulation'][dcid]
population = {
'date': pop_data['date'],
'data': [{
'dcid': dcid,
'data': {
'Count_Person': pop_data['value']
}
}],
'sources': [pop_data['metadata']['provenanceUrl']]
}
highlight = {gettext('CHART_TITLE-Population'): population}
response = {
'pageChart': spec_and_stat,
'allChildPlaces': get_i18n_all_child_places(raw_page_data),
'childPlacesType': raw_page_data.get('childPlacesType', ""),
'childPlaces': raw_page_data.get('childPlaces', []),
'parentPlaces': raw_page_data.get('parentPlaces', []),
'similarPlaces': raw_page_data.get('similarPlaces', []),
'nearbyPlaces': raw_page_data.get('nearbyPlaces', []),
'categories': categories,
'names': names,
'highlight': highlight,
}
return Response(json.dumps(response), 200, mimetype='application/json')
def homepage():
place_names = place_api.get_display_name('^'.join(_HOMEPAGE_PLACE_DCIDS),
g.locale)
return render_template('static/homepage.html', place_names=place_names)