def build_address_format_training_data(admin_rtree, language_rtree, neighborhoods_rtree, infile, out_dir, tag_components=True):
'''
Creates formatted address training data for supervised sequence labeling (or potentially
for unsupervised learning e.g. for word vectors) using addr:* tags in OSM.
Example:
cs cz Gorkého/road ev.2459/house_number | 40004/postcode Trmice/city | CZ/country
The field structure is similar to other training data created by this script i.e.
{language, country, data}. The data field here is a sequence of labeled tokens similar
to what we might see in part-of-speech tagging.
This format uses a special character "|" to denote possible breaks in the input (comma, newline).
Note that for the address parser, we'd like it to be robust to many different types
of input, so we may selectively eleminate components
This information can potentially be used downstream by the sequence model as these
breaks may be present at prediction time.
Example:
sr rs Crkva Svetog Arhangela Mihaila | Vukov put BB | 15303 Trsic
This may be useful in learning word representations, statistical phrases, morphology
or other models requiring only the sequence of words.
'''
i = 0
formatter = AddressFormatter()
osm_address_components.configure()
if tag_components:
formatted_tagged_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_TAGGED_FILENAME), 'w')
writer = csv.writer(formatted_tagged_file, 'tsv_no_quote')
else:
formatted_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_FILENAME), 'w')
writer = csv.writer(formatted_file, 'tsv_no_quote')
remove_keys = OSM_IGNORE_KEYS
for node_id, value, deps in parse_osm(infile):
try:
latitude, longitude = latlon_to_decimal(value['lat'], value['lon'])
except Exception:
continue
country, candidate_languages, language_props = country_and_languages(language_rtree, latitude, longitude)
if not (country and candidate_languages):
continue
for key in remove_keys:
_ = value.pop(key, None)
language = None
if tag_components:
if len(candidate_languages) == 1:
language = candidate_languages[0]['lang']
else:
street = value.get('addr:street', None)
if street is not None:
language = disambiguate_language(street, [(l['lang'], l['default']) for l in candidate_languages])
else:
language = UNKNOWN_LANGUAGE
address_components = {k: v for k, v in value.iteritems() if k in formatter.aliases}
formatter.replace_aliases(address_components)
address_country = address_components.get(AddressFormatter.COUNTRY)
'''
Country names
-------------
In OSM, addr:country is almost always an ISO-3166 alpha-2 country code.
However, we'd like to expand these to include natural language forms
of the country names we might be likely to encounter in a geocoder or
handwritten address.
These splits are somewhat arbitrary but could potentially be fit to data
from OpenVenues or other sources on the usage of country name forms.
If the address includes a country, the selection procedure proceeds as follows:
1. With probability a, select the country name in the language of the address
(determined above), or with the localized country name if the language is
undtermined or ambiguous.
2. With probability b(1-a), sample a language from the distribution of
languages on the Internet and use the country's name in that language.
3. This is implicit, but with probability (1-b)(1-a), keep the country code
'''
non_local_language = None
if random.random() < 0.3:
# 30% of the time: add Quattroshapes country
address_country = country.upper()
r = random.random()
# 1. 60% of the time: use the country name in the current language or the country's local language
if address_country and r < 0.6:
localized = None
if language and language not in (AMBIGUOUS_LANGUAGE, UNKNOWN_LANGUAGE):
localized = language_country_names.get(language, {}).get(address_country.upper())
if not localized:
localized = country_localized_display_name(address_country.lower())
if localized:
address_components[AddressFormatter.COUNTRY] = localized
# 2. 10% of the time: country's name in a language samples from the distribution of languages on the Internet
elif address_country and r < 0.7:
non_local_language = sample_random_language()
lang_country = language_country_names.get(non_local_language, {}).get(address_country.upper())
if lang_country:
address_components[AddressFormatter.COUNTRY] = lang_country
# 3. Implicit: the rest of the time keep the country code
'''
States
------
Primarily for the US, Canada and Australia, OSM tends to use the abbreviated state name
whereas we'd like to include both forms, so wtih some probability, replace the abbreviated
name with the unabbreviated one e.g. CA => California
'''
address_state = address_components.get(AddressFormatter.STATE)
if address_state:
state_full_name = STATE_ABBREVIATIONS.get(country.upper(), {}).get(address_state.upper(), {}).get(language)
if state_full_name and random.random() < 0.3:
address_components[AddressFormatter.STATE] = state_full_name
'''
OSM boundaries
--------------
For many addresses, the city, district, region, etc. are all implicitly
generated by the reverse geocoder e.g. we do not need an addr:city tag
to identify that 40.74, -74.00 is in New York City as well as its parent
geographies (New York county, New York state, etc.)
Where possible we augment the addr:* tags with some of the reverse-geocoded
relations from OSM.
Since addresses found on the web may have the same properties, we
include these qualifiers in the training data.
'''
osm_components = osm_reverse_geocoded_components(admin_rtree, country, latitude, longitude)
if osm_components:
if non_local_language is not None:
suffix = ':{}'.format(non_local_language)
else:
suffix = ''
name_key = ''.join(('name', suffix))
raw_name_key = 'name'
short_name_key = ''.join(('short_name', suffix))
raw_short_name_key = 'short_name'
simple_name_key = 'name:simple'
international_name_key = 'int_name'
alt_name_key = ''.join(('alt_name', suffix))
raw_alt_name_key = 'alt_name'
official_name_key = ''.join(('official_name', suffix))
raw_official_name_key = 'official_name'
iso_code_key = 'ISO3166-1:alpha2'
iso_code3_key = 'ISO3166-1:alpha3'
poly_components = defaultdict(list)
for component, components_values in osm_components.iteritems():
seen = set()
# Choose which name to use with given probabilities
r = random.random()
if r < 0.7:
# 70% of the time use the name tag
key = name_key
raw_key = raw_name_key
elif r < 0.8:
# 10% of the time use the short name
key = short_name_key
raw_key = raw_short_name_key
elif r < 0.9:
# 10% of the time use the official name
key = official_name_key
raw_key = raw_official_name_key
else:
# 10% of the time use the official name
key = alt_name_key
raw_key = raw_alt_name_key
for component_value in components_values:
r = random.random()
name = None
if iso_code3_key in component_value and r < 0.1:
name = component_value[iso_code3_key]
elif iso_code_key in component_value and r < 0.3:
name = component_value[iso_code_key]
elif language == 'en' and not non_local_language and r < 0.7:
# Particularly to address the US (prefer United States,
# not United States of America) but may capture variations
# in other English-speaking countries as well.
if simple_name_key in component_value:
name = component_value[simple_name_key]
elif international_name_key in component_value:
name = component_value[international_name_key]
if not name:
name = component_value.get(key, component_value.get(raw_key))
if not name:
name = component_value.get(name_key, component_value.get(raw_name_key))
if not name:
continue
if (component, name) not in seen:
poly_components[component].append(name)
seen.add((component, name))
for component, vals in poly_components.iteritems():
if component not in address_components or non_local_language:
val = u', '.join(vals)
if component == AddressFormatter.STATE and random.random() < 0.7:
val = STATE_EXPANSIONS.get(address_country, {}).get(val, val)
address_components[component] = val
'''
Neighborhoods
-------------
In some cities, neighborhoods may be included in a free-text address.
OSM includes many neighborhoods but only as points, rather than the polygons
needed to perform reverse-geocoding. We use a hybrid index containing
Quattroshapes/Zetashapes polygons matched fuzzily with OSM names (which are
on the whole of better quality).
'''
neighborhood = neighborhoods_rtree.point_in_poly(latitude, longitude)
if neighborhood and AddressFormatter.SUBURB not in address_components:
address_components[AddressFormatter.SUBURB] = neighborhood['name']
# Version with all components
formatted_address = formatter.format_address(country, address_components, tag_components=tag_components, minimal_only=not tag_components)
if tag_components:
formatted_addresses = []
formatted_addresses.append(formatted_address)
address_components = {k: v for k, v in address_components.iteritems() if k in OSM_ADDRESS_COMPONENT_VALUES}
if not address_components:
continue
current_components = component_bitset(address_components.keys())
for component in address_components.keys():
if current_components ^ OSM_ADDRESS_COMPONENT_VALUES[component] in OSM_ADDRESS_COMPONENTS_VALID and random.random() < 0.5:
address_components.pop(component)
current_components ^= OSM_ADDRESS_COMPONENT_VALUES[component]
if not address_components:
break
formatted_address = formatter.format_address(country, address_components, tag_components=tag_components, minimal_only=False)
formatted_addresses.append(formatted_address)
for formatted_address in formatted_addresses:
if formatted_address and formatted_address.strip():
formatted_address = tsv_string(formatted_address)
if not formatted_address or not formatted_address.strip():
continue
row = (language, country, formatted_address)
writer.writerow(row)
elif formatted_address and formatted_address.strip():
formatted_address = tsv_string(formatted_address)
writer.writerow([formatted_address])
i += 1
if i % 1000 == 0 and i > 0:
print 'did', i, 'formatted addresses'
def build_address_format_training_data(admin_rtree, language_rtree, neighborhoods_rtree, quattroshapes_rtree, geonames, infile, out_dir, tag_components=True):
'''
Creates formatted address training data for supervised sequence labeling (or potentially
for unsupervised learning e.g. for word vectors) using addr:* tags in OSM.
Example:
cs cz Gorkého/road ev.2459/house_number | 40004/postcode Trmice/city | CZ/country
The field structure is similar to other training data created by this script i.e.
{language, country, data}. The data field here is a sequence of labeled tokens similar
to what we might see in part-of-speech tagging.
This format uses a special character "|" to denote possible breaks in the input (comma, newline).
Note that for the address parser, we'd like it to be robust to many different types
of input, so we may selectively eleminate components
This information can potentially be used downstream by the sequence model as these
breaks may be present at prediction time.
Example:
sr rs Crkva Svetog Arhangela Mihaila | Vukov put BB | 15303 Trsic
This may be useful in learning word representations, statistical phrases, morphology
or other models requiring only the sequence of words.
'''
i = 0
formatter = AddressFormatter()
osm_address_components.configure()
if tag_components:
formatted_tagged_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_TAGGED_FILENAME), 'w')
writer = csv.writer(formatted_tagged_file, 'tsv_no_quote')
else:
formatted_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_FILENAME), 'w')
writer = csv.writer(formatted_file, 'tsv_no_quote')
remove_keys = OSM_IGNORE_KEYS
alpha3_codes = {c.alpha2: c.alpha3 for c in pycountry.countries}
for node_id, value, deps in parse_osm(infile):
try:
latitude, longitude = latlon_to_decimal(value['lat'], value['lon'])
except Exception:
continue
country, candidate_languages, language_props = country_and_languages(language_rtree, latitude, longitude)
if not (country and candidate_languages):
continue
for key in remove_keys:
_ = value.pop(key, None)
language = None
more_than_one_official_language = len(candidate_languages) > 1
if tag_components:
if len(candidate_languages) == 1:
language = candidate_languages[0]['lang']
else:
street = value.get('addr:street', None)
namespaced = [l['lang'] for l in candidate_languages if 'addr:street:{}'.format(l['lang']) in value]
if street is not None and not namespaced:
language = disambiguate_language(street, [(l['lang'], l['default']) for l in candidate_languages])
elif namespaced and random.random() < 0.6:
language = random.choice(namespaced)
lang_suffix = ':{}'.format(language)
for k in value:
if k.startswith('addr:') and k.endswith(lang_suffix):
value[k.rstrip(lang_suffix)] = value[k]
else:
language = UNKNOWN_LANGUAGE
address_components = {k: v for k, v in value.iteritems() if k in formatter.aliases}
formatter.replace_aliases(address_components)
address_country = address_components.get(AddressFormatter.COUNTRY)
'''
Country names
-------------
In OSM, addr:country is almost always an ISO-3166 alpha-2 country code.
However, we'd like to expand these to include natural language forms
of the country names we might be likely to encounter in a geocoder or
handwritten address.
These splits are somewhat arbitrary but could potentially be fit to data
from OpenVenues or other sources on the usage of country name forms.
If the address includes a country, the selection procedure proceeds as follows:
1. With probability a, select the country name in the language of the address
(determined above), or with the localized country name if the language is
undtermined or ambiguous.
2. With probability b(1-a), sample a language from the distribution of
languages on the Internet and use the country's name in that language.
3. This is implicit, but with probability (1-b)(1-a), keep the country code
'''
non_local_language = None
if random.random() < 0.3:
# 30% of the time: add Quattroshapes country
address_country = country.upper()
r = random.random()
# 1. 60% of the time: use the country name in the current language or the country's local language
if address_country and r < 0.6:
localized = None
if language and language not in (AMBIGUOUS_LANGUAGE, UNKNOWN_LANGUAGE):
localized = language_country_names.get(language, {}).get(address_country.upper())
if not localized:
localized = country_localized_display_name(address_country.lower())
if localized:
address_components[AddressFormatter.COUNTRY] = localized
# 2. 10% of the time: country's name in a language samples from the distribution of languages on the Internet
elif address_country and r < 0.7:
non_local_language = sample_random_language()
lang_country = language_country_names.get(non_local_language, {}).get(address_country.upper())
if lang_country:
address_components[AddressFormatter.COUNTRY] = lang_country
# 3. 10% of the time: use the country's alpha-3 ISO code
elif address_country and r < 0.8:
iso_code_alpha3 = alpha3_codes.get(address_country)
if iso_code_alpha3:
address_components[AddressFormatter.COUNTRY] = iso_code_alpha3
# 4. Implicit: the rest of the time keep the alpha-2 country code
'''
Venue names
-----------
Some venues have multiple names listed in OSM, grab them all
'''
venue_names = []
for key in ('name', 'alt_name', 'loc_name', 'int_name', 'old_name'):
venue_name = value.get(key)
if venue_name:
venue_names.append(venue_name)
'''
States
------
Primarily for the US, Canada and Australia, OSM tends to use the abbreviated state name
whereas we'd like to include both forms, so wtih some probability, replace the abbreviated
name with the unabbreviated one e.g. CA => California
'''
address_state = address_components.get(AddressFormatter.STATE)
if address_state and not non_local_language:
state_full_name = STATE_ABBREVIATIONS.get(country.upper(), {}).get(address_state.upper(), {}).get(language)
if state_full_name and random.random() < 0.3:
address_components[AddressFormatter.STATE] = state_full_name
elif address_state and non_local_language:
_ = address_components.pop(AddressFormatter.STATE, None)
'''
OSM boundaries
--------------
For many addresses, the city, district, region, etc. are all implicitly
generated by the reverse geocoder e.g. we do not need an addr:city tag
to identify that 40.74, -74.00 is in New York City as well as its parent
geographies (New York county, New York state, etc.)
Where possible we augment the addr:* tags with some of the reverse-geocoded
relations from OSM.
Since addresses found on the web may have the same properties, we
include these qualifiers in the training data.
'''
osm_components = osm_reverse_geocoded_components(admin_rtree, country, latitude, longitude)
if non_local_language is not None:
osm_suffix = ':{}'.format(non_local_language)
elif more_than_one_official_language and language not in (AMBIGUOUS_LANGUAGE, UNKNOWN_LANGUAGE):
osm_suffix = ':{}'.format(language)
else:
osm_suffix = ''
name_key = ''.join(('name', osm_suffix))
raw_name_key = 'name'
simple_name_key = 'name:simple'
international_name_key = 'int_name'
iso_code_key = 'ISO3166-1:alpha2'
iso_code3_key = 'ISO3166-1:alpha3'
if osm_components:
poly_components = defaultdict(list)
existing_city_name = address_components.get(AddressFormatter.CITY)
for component, components_values in osm_components.iteritems():
seen = set()
key, raw_key = osm_pick_random_name_key(suffix=osm_suffix)
for component_value in components_values:
r = random.random()
name = None
if iso_code3_key in component_value and r < 0.1:
name = component_value[iso_code3_key]
elif iso_code_key in component_value and r < 0.3:
name = component_value[iso_code_key]
elif language == 'en' and not non_local_language and r < 0.7:
# Particularly to address the US (prefer United States,
# not United States of America) but may capture variations
# in other English-speaking countries as well.
if simple_name_key in component_value:
name = component_value[simple_name_key]
elif international_name_key in component_value:
name = component_value[international_name_key]
if not name:
name = component_value.get(key, component_value.get(raw_key))
if not name or (component != AddressFormatter.CITY and name == existing_city_name):
name = component_value.get(name_key, component_value.get(raw_name_key))
if not name or (component != AddressFormatter.CITY and name == existing_city_name):
continue
if (component, name) not in seen:
poly_components[component].append(name)
seen.add((component, name))
for component, vals in poly_components.iteritems():
if component not in address_components or (non_local_language and random.random() < 0.4):
if component == AddressFormatter.STATE_DISTRICT and random.random() < 0.5:
num = random.randrange(1, len(vals) + 1)
val = u', '.join(vals[:num])
else:
val = random.choice(vals)
if component == AddressFormatter.STATE and random.random() < 0.7:
val = STATE_EXPANSIONS.get(address_country, {}).get(val, val)
address_components[component] = val
'''
Quattroshapes/GeoNames cities
-----------------------------
Quattroshapes isn't great for everything, but it has decent city boundaries
in places where OSM sometimes does not (or at least in places where we aren't
currently able to create valid polygons). While Quattroshapes itself doesn't
reliably use local names, which we'll want for consistency
'''
if non_local_language or (AddressFormatter.CITY not in address_components and random.random() < 0.2):
lang = non_local_language or language
quattroshapes_cities = quattroshapes_rtree.point_in_poly(latitude, longitude, return_all=True)
for result in quattroshapes_cities:
if result.get(quattroshapes_rtree.LEVEL) == quattroshapes_rtree.LOCALITY and quattroshapes_rtree.GEONAMES_ID in result:
geonames_id = int(result[quattroshapes_rtree.GEONAMES_ID].split(',')[0])
names = geonames.get_alternate_names(geonames_id)
if not names or lang not in names:
continue
city = None
if 'abbr' not in names or non_local_language:
# Use the common city name in the target language
city = names[lang][0][0]
elif random.random() < 0.1:
# Use an abbreviation: NYC, BK, SF, etc.
city = random.choice(names['abbr'])[0]
if not city or not city.strip():
continue
address_components[AddressFormatter.CITY] = city
break
else:
if non_local_language and AddressFormatter.CITY in address_components and (
AddressFormatter.CITY_DISTRICT in osm_components or
AddressFormatter.SUBURB in osm_components):
address_components.pop(AddressFormatter.CITY)
'''
Neighborhoods
-------------
In some cities, neighborhoods may be included in a free-text address.
OSM includes many neighborhoods but only as points, rather than the polygons
needed to perform reverse-geocoding. We use a hybrid index containing
Quattroshapes/Zetashapes polygons matched fuzzily with OSM names (which are
on the whole of better quality).
'''
neighborhoods = neighborhoods_rtree.point_in_poly(latitude, longitude, return_all=True)
neighborhood_levels = defaultdict(list)
for neighborhood in neighborhoods:
place_type = neighborhood.get('place')
polygon_type = neighborhood.get('polygon_type')
key, raw_key = osm_pick_random_name_key(suffix=osm_suffix)
name = neighborhood.get(key, neighborhood.get(raw_key))
if not name:
name = neighborhood.get(name_key, neighborhood.get(raw_name_key))
name_prefix = neighborhood.get('name:prefix')
if name_prefix and random.random() < 0.5:
name = u' '.join([name_prefix, name])
if not name:
continue
neighborhood_level = AddressFormatter.SUBURB
if place_type == 'borough' or polygon_type == 'local_admin':
neighborhood_level = AddressFormatter.CITY_DISTRICT
# Optimization so we don't use e.g. Brooklyn multiple times
city_name = address_components.get(AddressFormatter.CITY)
if name == city_name:
name = neighborhood.get(name_key, neighborhood.get(raw_name_key))
if not name or name == city_name:
continue
neighborhood_levels[neighborhood_level].append(name)
for component, neighborhoods in neighborhood_levels.iteritems():
if component not in address_components and random.random() < 0.5:
address_components[component] = neighborhoods[0]
'''
Name normalization
------------------
Probabilistically strip standard prefixes/suffixes e.g. "London Borough of"
'''
for component in BOUNDARY_COMPONENTS:
name = address_components.get(component)
if not name:
continue
replacement = replace_name_prefixes(replace_name_suffixes(name))
if replacement != name and random.random() < 0.6:
address_components[component] = replacement
'''
Name deduping
-------------
For some cases like "Antwerpen, Antwerpen, Antwerpen"
that are very unlikely to occur in real life.
'''
name_components = defaultdict(list)
for component in (AddressFormatter.STATE_DISTRICT, AddressFormatter.CITY, AddressFormatter.CITY_DISTRICT, AddressFormatter.SUBURB):
name = address_components.get(component)
if name:
name_components[name].append(component)
for name, components in name_components.iteritems():
if len(components) > 1:
for component in components[1:]:
address_components.pop(component, None)
'''
House number cleanup
--------------------
For some OSM nodes, particularly in Uruguay, we get house numbers
that are actually a comma-separated list.
If there's one comma in the house number, allow it as it might
be legitimate, but if there are 2 or more, just take the first one.
'''
house_number = address_components.get(AddressFormatter.HOUSE_NUMBER)
if house_number and house_number.count(',') >= 2:
for num in house_number.split(','):
num = num.strip()
if num:
address_components[AddressFormatter.HOUSE_NUMBER] = num
break
else:
address_components.pop(AddressFormatter.HOUSE_NUMBER, None)
# Version with all components
formatted_address = formatter.format_address(country, address_components, tag_components=tag_components, minimal_only=not tag_components)
if tag_components:
formatted_addresses = []
formatted_addresses.append(formatted_address)
seen = set([formatted_address])
address_components = {k: v for k, v in address_components.iteritems() if k in OSM_ADDRESS_COMPONENT_VALUES}
if not address_components:
continue
current_components = address_components.keys()
random.shuffle(current_components)
component_set = component_bitset(address_components.keys())
for component in current_components:
if component_set ^ OSM_ADDRESS_COMPONENT_VALUES[component] in OSM_ADDRESS_COMPONENTS_VALID and random.random() < 0.5:
address_components.pop(component)
component_set ^= OSM_ADDRESS_COMPONENT_VALUES[component]
if not address_components:
break
# Since venue names are 1-per-record, we must use them all
for venue_name in (venue_names or [None]):
if venue_name and AddressFormatter.HOUSE in address_components:
address_components[AddressFormatter.HOUSE] = venue_name
formatted_address = formatter.format_address(country, address_components, tag_components=tag_components, minimal_only=False)
if formatted_address not in seen:
formatted_addresses.append(formatted_address)
seen.add(formatted_address)
for formatted_address in formatted_addresses:
if formatted_address and formatted_address.strip():
formatted_address = tsv_string(formatted_address)
if not formatted_address or not formatted_address.strip():
continue
row = (language, country, formatted_address)
writer.writerow(row)
elif formatted_address and formatted_address.strip():
formatted_address = tsv_string(formatted_address)
writer.writerow([formatted_address])
i += 1
if i % 1000 == 0 and i > 0:
print 'did', i, 'formatted addresses'