def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR):
languages = scrape_all_nominatim_category_pages(url=url)
for lang, phrases in six.iteritems(languages):
filename = os.path.join(output_dir, '{}.tsv'.format(lang.lower()))
with open(filename, 'w') as f:
writer = csv.writer(f, delimiter='\t')
writer.writerow(('key', 'value', 'is_plural', 'phrase'))
for phrase, key, value, is_plural in phrases:
writer.writerow((safe_encode(key), safe_encode(value),
str(int(is_plural)), safe_encode(phrase)))
print('Done')
def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR):
languages = scrape_all_nominatim_category_pages(url=url)
for lang, phrases in six.iteritems(languages):
filename = os.path.join(output_dir, '{}.tsv'.format(lang.lower()))
with open(filename, 'w') as f:
writer = csv.writer(f, delimiter='\t')
writer.writerow(('key', 'value', 'is_plural', 'phrase'))
for phrase, key, value, is_plural in phrases:
writer.writerow((safe_encode(key), safe_encode(value),
str(int(is_plural)), safe_encode(phrase)))
print('Done')
def write_to_tsv(self, out_filename, min_threshold=5):
writer = csv.writer(open(out_filename, 'w'), delimiter='\t')
for k, v in self.names_lower.most_common():
if v < min_threshold:
break
canonical = self.chain_canonical.get(k)
if canonical:
canonical = self.names_cap[canonical].most_common(1)[0][0]
else:
canonical = ''
most_common_cap = self.names_cap[k].most_common(1)[0][0]
writer.writerow((safe_encode(k), safe_encode(most_common_cap),
safe_encode(canonical),
safe_encode(1) if k in self.all_chains else '',
safe_encode(v)))
def write_to_tsv(self, out_filename, min_threshold=5):
writer = csv.writer(open(out_filename, 'w'), delimiter='\t')
for k, v in self.names_lower.most_common():
if v < min_threshold:
break
canonical = self.chain_canonical.get(k)
if canonical:
canonical = self.names_cap[canonical].most_common(1)[0][0]
else:
canonical = ''
most_common_cap = self.names_cap[k].most_common(1)[0][0]
writer.writerow((safe_encode(k),
safe_encode(most_common_cap),
safe_encode(canonical),
safe_encode(1) if k in self.all_chains else '',
safe_encode(v)))
def __init__(self, *dictionaries):
self.dictionaries = dictionaries
self.canonicals = {}
kvs = defaultdict(OrderedDict)
for language in address_phrase_dictionaries.languages:
for dictionary_name in self.dictionaries:
is_suffix_dictionary = 'suffixes' in dictionary_name
is_prefix_dictionary = 'prefixes' in dictionary_name
for phrases in address_phrase_dictionaries.phrases.get((language, dictionary_name), []):
canonical = phrases[0]
canonical_normalized = normalize_string(canonical)
self.canonicals[(canonical, language, dictionary_name)] = phrases[1:]
for i, phrase in enumerate(phrases):
if phrase in POSSIBLE_ROMAN_NUMERALS:
continue
is_canonical = normalize_string(phrase) == canonical_normalized
if is_suffix_dictionary:
phrase = SUFFIX_KEY + phrase[::-1]
elif is_prefix_dictionary:
phrase = PREFIX_KEY + phrase
kvs[phrase][(language, dictionary_name, canonical)] = is_canonical
kvs = [(k, '|'.join([l, d, str(int(i)), safe_encode(c)])) for k, vals in kvs.iteritems() for (l, d, c), i in vals.iteritems()]
self.trie = BytesTrie(kvs)
def create_intersections(self, outfile):
out = open(outfile, 'w')
for node_id, node_props, ways in self.intersections():
d = {'id': safe_encode(node_id),
'node': node_props,
'ways': ways}
out.write(json.dumps(d) + six.u('\n'))
def to_tsv(self, filename, mode='w', max_rows=None):
f = open(filename, mode)
writer = csv.writer(f, delimiter='\t')
for i, (k, v) in enumerate(Counter(self.vocab).most_common()):
if max_rows is not None and i == max_rows:
break
gram = []
for t, c in k:
gram.append(t)
if c != token_types.IDEOGRAPHIC_CHAR:
gram.append(six.text_type(' '))
phrase = six.text_type('').join(gram)
writer.writerow((safe_encode(phrase), safe_encode(len(k)), safe_encode(v)))
def openaddresses_download_configured_files(out_dir):
for path in openaddresses_config.sources:
source = six.b('/').join([safe_encode(p) for p in path])
filename = safe_encode(path[-1]) + six.b('.zip')
zip_path = filename + '.zip'
zip_url_path = six.b('/').join([safe_encode(p) for p in path[:-1]] + [quote_plus(filename)])
url = urljoin(OPENADDRESSES_LATEST_DIR, zip_url_path)
download_pre_release_downloads(out_dir)
print(six.u('doing {}').format(safe_decode(source)))
success = download_and_unzip_file(url, out_dir)
if not success:
print(six.u('ERR: could not download {}').format(source))
def to_tsv(self, filename, mode='w', max_rows=None):
f = open(filename, mode)
writer = csv.writer(f, delimiter='\t')
for i, (k, v) in enumerate(Counter(self.vocab).most_common()):
if max_rows is not None and i == max_rows:
break
gram = []
for t, c in k:
gram.append(t)
if c != token_types.IDEOGRAPHIC_CHAR:
gram.append(six.text_type(' '))
phrase = six.text_type('').join(gram)
writer.writerow(
(safe_encode(phrase), safe_encode(len(k)), safe_encode(v)))
def openaddresses_download_configured_files(out_dir):
for path in openaddresses_config.sources:
source = six.b('/').join([safe_encode(p) for p in path])
filename = safe_encode(path[-1]) + six.b('.zip')
zip_path = filename + '.zip'
zip_url_path = six.b('/').join([safe_encode(p) for p in path[:-1]] +
[quote_plus(filename)])
url = urljoin(OPENADDRESSES_LATEST_DIR, zip_url_path)
download_pre_release_downloads(out_dir)
print(six.u('doing {}').format(safe_decode(source)))
success = download_and_unzip_file(url, out_dir)
if not success:
print(six.u('ERR: could not download {}').format(source))
def name_key_dist(self, props, component):
object_type = props.get('type')
object_id = safe_encode(props.get('id', ''))
if (object_type, object_id) in self.exceptions:
values, probs = self.exceptions[(object_type, object_id)]
return values, probs
name_keys, probs = self.component_name_keys.get(component, (self.name_keys, self.name_key_probs))
return name_keys, probs
def include_component_simple(self, component, containing_ids, country=None):
containing = self.get_property(('components', component, 'containing'), country=country, default=None)
if containing is not None:
for c in containing:
if (c['type'], safe_encode(c['id'])) in containing_ids:
return random.random() < c['probability']
probability = self.get_property(('components', component, 'probability'), country=country, default=0.0)
return random.random() < probability
def name_key_dist(self, props, component):
object_type = props.get('type')
object_id = safe_encode(props.get('id', ''))
if (object_type, object_id) in self.exceptions:
values, probs = self.exceptions[(object_type, object_id)]
return values, probs
name_keys, probs = self.component_name_keys.get(
component, (self.name_keys, self.name_key_probs))
return name_keys, probs
def get_script_codes(all_scripts):
if not os.path.exists(LOCAL_ISO_15924_FILE):
temp_dir = tempfile.gettempdir()
script_codes_filename = os.path.join(temp_dir,
ISO_15924_URL.rsplit('/')[-1])
# This comes as a .zip
script_codes_response = requests.get(ISO_15924_URL)
zf = ZipFile(StringIO(script_codes_response.content))
iso15924_filename = [
name for name in zf.namelist() if name.startswith('iso15924')
][0]
# Strip out the comments, etc.
temp_iso15924_file = u'\n'.join([
line.rstrip()
for line in safe_decode(zf.read(iso15924_filename)).split('\n')
if line.strip() and not line.strip().startswith('#')
])
f = open(LOCAL_ISO_15924_FILE, 'w')
f.write(safe_encode(temp_iso15924_file))
f.close()
script_codes_file = open(LOCAL_ISO_15924_FILE)
script_codes = {}
seen_scripts = set()
# Scripts in the CLDR repos use 4-letter ISO-15924 codes, so map those
for code, _, name, _, _, _ in csv.reader(script_codes_file, delimiter=';'):
if name in all_scripts:
script_codes[code] = name
seen_scripts.add(name)
else:
normalized_name = name.split('(')[0].strip()
if normalized_name in all_scripts and normalized_name not in seen_scripts:
script_codes[code] = normalized_name
seen_scripts.add(normalized_name)
value_aliases = get_property_value_aliases()
script_aliases = value_aliases['sc']
for code, script in script_aliases.iteritems():
if code not in script_codes and script in all_scripts:
script_codes[code] = script
script_codes.update(SCRIPT_ALIASES_SUPPLEMENTAL)
return script_codes
def parse_osm(filename, allowed_types=ALL_OSM_TAGS, dependencies=False):
'''
Parse a file in .osm format iteratively, generating tuples like:
('node:1', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:2', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:3', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:4', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('way:4444', OrderedDict([('name', 'Main Street')]), [1,2,3,4])
'''
f = open(filename)
parser = etree.iterparse(f)
single_type = len(allowed_types) == 1
for (_, elem) in parser:
elem_id = long(elem.attrib.pop('id', 0))
item_type = elem.tag
if elem_id >= WAY_OFFSET and elem_id < RELATION_OFFSET:
elem_id -= WAY_OFFSET
item_type = 'way'
elif elem_id >= RELATION_OFFSET:
elem_id -= RELATION_OFFSET
item_type = 'relation'
if item_type in allowed_types:
attrs = OrderedDict(elem.attrib)
attrs['type'] = item_type
attrs['id'] = safe_encode(elem_id)
top_level_attrs = set(attrs)
deps = [] if dependencies else None
for e in elem.getchildren():
if e.tag == 'tag':
# Prevent user-defined lat/lon keys from overriding the lat/lon on the node
key = e.attrib['k']
if key not in top_level_attrs:
attrs[key] = e.attrib['v']
elif dependencies and item_type == 'way' and e.tag == 'nd':
deps.append(long(e.attrib['ref']))
elif dependencies and item_type == 'relation' and e.tag == 'member' and 'role' in e.attrib:
deps.append((long(e.attrib['ref']), e.attrib.get('type'),
e.attrib['role']))
key = elem_id if single_type else '{}:{}'.format(
item_type, elem_id)
yield key, attrs, deps
if elem.tag in ALL_OSM_TAGS:
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
def configure(self, base_dir=DICTIONARIES_DIR):
kvs = defaultdict(OrderedDict)
for lang in os.listdir(DICTIONARIES_DIR):
for filename in self.dictionaries:
is_suffix_dictionary = 'suffixes' in filename
is_prefix_dictionary = 'prefixes' in filename
dictionary_name = filename.split('.', 1)[0]
path = os.path.join(DICTIONARIES_DIR, lang, filename)
if not os.path.exists(path):
continue
for line in open(path):
line = line.strip()
if not line:
continue
phrases = safe_decode(line).split(u'|')
if not phrases:
continue
canonical = phrases[0]
canonical_normalized = normalize_string(canonical)
self.canonicals[(canonical, lang,
dictionary_name)] = phrases[1:]
for i, phrase in enumerate(phrases):
if phrase in POSSIBLE_ROMAN_NUMERALS:
continue
is_canonical = normalize_string(
phrase) == canonical_normalized
if is_suffix_dictionary:
phrase = SUFFIX_KEY + phrase[::-1]
elif is_prefix_dictionary:
phrase = PREFIX_KEY + phrase
kvs[phrase][(lang, dictionary_name,
canonical)] = is_canonical
kvs = [(k, '|'.join([l, d, str(int(i)),
safe_encode(c)])) for k, vals in kvs.iteritems()
for (l, d, c), i in vals.iteritems()]
self.trie = BytesTrie(kvs)
self.configured = True
def parse_osm(filename, allowed_types=ALL_OSM_TAGS, dependencies=False):
'''
Parse a file in .osm format iteratively, generating tuples like:
('node:1', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:2', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:3', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('node:4', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
('way:4444', OrderedDict([('name', 'Main Street')]), [1,2,3,4])
'''
f = open(filename)
parser = etree.iterparse(f)
single_type = len(allowed_types) == 1
for (_, elem) in parser:
elem_id = long(elem.attrib.pop('id', 0))
item_type = elem.tag
if elem_id >= WAY_OFFSET and elem_id < RELATION_OFFSET:
elem_id -= WAY_OFFSET
item_type = 'way'
elif elem_id >= RELATION_OFFSET:
elem_id -= RELATION_OFFSET
item_type = 'relation'
if item_type in allowed_types:
attrs = OrderedDict(elem.attrib)
attrs['type'] = item_type
attrs['id'] = safe_encode(elem_id)
top_level_attrs = set(attrs)
deps = [] if dependencies else None
for e in elem.getchildren():
if e.tag == 'tag':
# Prevent user-defined lat/lon keys from overriding the lat/lon on the node
key = e.attrib['k']
if key not in top_level_attrs:
attrs[key] = e.attrib['v']
elif dependencies and item_type == 'way' and e.tag == 'nd':
deps.append(long(e.attrib['ref']))
elif dependencies and item_type == 'relation' and e.tag == 'member' and 'role' in e.attrib:
deps.append((long(e.attrib['ref']), e.attrib.get('type'), e.attrib['role']))
key = elem_id if single_type else '{}:{}'.format(item_type, elem_id)
yield key, attrs, deps
if elem.tag in ALL_OSM_TAGS:
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
def __init__(self, boundaries_dir=OSM_BOUNDARIES_DIR):
self.config = {}
self.use_admin_center = {}
for filename in os.listdir(boundaries_dir):
if not filename.endswith('.yaml'):
continue
country_code = filename.rsplit('.yaml', 1)[0]
data = yaml.load(open(os.path.join(boundaries_dir, filename)))
for prop, values in six.iteritems(data):
if not hasattr(values, 'items'):
# non-dict key
continue
for k, v in values.iteritems():
if isinstance(
v, six.string_types
) and v not in AddressFormatter.address_formatter_fields:
raise ValueError(
u'Invalid value in {} for prop={}, key={}: {}'.
format(filename, prop, k, v))
if prop == 'overrides':
self.use_admin_center.update({
(r['type'], safe_encode(r['id'])):
r.get('probability', 1.0)
for r in values.get('use_admin_center', [])
})
containing_overrides = values.get('contained_by', {})
if not containing_overrides:
continue
for id_type, vals in six.iteritems(containing_overrides):
for element_id in vals:
override_config = vals[element_id]
config = deepcopy(data)
config.pop('overrides')
recursive_merge(config, override_config)
vals[element_id] = config
self.config[country_code] = data
def download_dependencies(self, path):
data = json.load(open(path))
props = data['properties']
_, filename = os.path.split(path)
current_wof_id = filename.rsplit('.geojson', 1)[0]
for hierarchy in props.get('wof:hierarchy', []):
for key, wof_id in six.iteritems(hierarchy):
wof_id = safe_encode(wof_id)
if wof_id != current_wof_id and wof_id != '-1' and not self.client.exists_locally(wof_id):
if not self.client.download_file(wof_id):
print('error downloading {}'.format(wof_id))
continue
return props.get('name')
def get_script_codes(all_scripts):
if not os.path.exists(LOCAL_ISO_15924_FILE):
temp_dir = tempfile.gettempdir()
script_codes_filename = os.path.join(temp_dir, ISO_15924_URL.rsplit('/')[-1])
# This comes as a .zip
script_codes_response = requests.get(ISO_15924_URL)
zf = ZipFile(StringIO(script_codes_response.content))
iso15924_filename = [name for name in zf.namelist() if name.startswith('iso15924')][0]
# Strip out the comments, etc.
temp_iso15924_file = u'\n'.join([line.rstrip() for line in safe_decode(zf.read(iso15924_filename)).split('\n')
if line.strip() and not line.strip().startswith('#')])
f = open(LOCAL_ISO_15924_FILE, 'w')
f.write(safe_encode(temp_iso15924_file))
f.close()
script_codes_file = open(LOCAL_ISO_15924_FILE)
script_codes = {}
seen_scripts = set()
# Scripts in the CLDR repos use 4-letter ISO-15924 codes, so map those
for code, _, name, _, _, _ in csv.reader(script_codes_file, delimiter=';'):
if name in all_scripts:
script_codes[code] = name
seen_scripts.add(name)
else:
normalized_name = name.split('(')[0].strip()
if normalized_name in all_scripts and normalized_name not in seen_scripts:
script_codes[code] = normalized_name
seen_scripts.add(normalized_name)
value_aliases = get_property_value_aliases()
script_aliases = value_aliases['sc']
for code, script in script_aliases.iteritems():
if code not in script_codes and script in all_scripts:
script_codes[code] = script
script_codes.update(SCRIPT_ALIASES_SUPPLEMENTAL)
return script_codes
def get_wikipedia_titles(db):
d = defaultdict(dict)
cursor = db.execute(wikipedia_query)
while True:
batch = cursor.fetchmany(BATCH_SIZE)
if not batch:
break
for (url, geonames_id, is_preferred) in batch:
title = normalize_wikipedia_url(safe_encode(url))
if title is not None and title.strip():
title = utf8_normalize(normalize_name(title))
d[title.lower()][geonames_id] = int(is_preferred or 0)
return d
def download_dependencies(self, path):
data = json.load(open(path))
props = data['properties']
_, filename = os.path.split(path)
current_wof_id = filename.rsplit('.geojson', 1)[0]
for hierarchy in props.get('wof:hierarchy', []):
for key, wof_id in six.iteritems(hierarchy):
wof_id = safe_encode(wof_id)
if wof_id != current_wof_id and wof_id != '-1' and not self.client.exists_locally(
wof_id):
if not self.client.download_file(wof_id):
print('error downloading {}'.format(wof_id))
continue
return props.get('name')
def configure(self, base_dir=DICTIONARIES_DIR):
kvs = defaultdict(OrderedDict)
for lang in os.listdir(DICTIONARIES_DIR):
for filename in self.dictionaries:
is_suffix_dictionary = 'suffixes' in filename
is_prefix_dictionary = 'prefixes' in filename
dictionary_name = filename.split('.', 1)[0]
path = os.path.join(DICTIONARIES_DIR, lang, filename)
if not os.path.exists(path):
continue
for line in open(path):
line = line.strip()
if not line:
continue
phrases = safe_decode(line).split(u'|')
if not phrases:
continue
canonical = phrases[0]
canonical_normalized = normalize_string(canonical)
self.canonicals[(canonical, lang, dictionary_name)] = phrases[1:]
for i, phrase in enumerate(phrases):
if phrase in POSSIBLE_ROMAN_NUMERALS:
continue
is_canonical = normalize_string(phrase) == canonical_normalized
if is_suffix_dictionary:
phrase = SUFFIX_KEY + phrase[::-1]
elif is_prefix_dictionary:
phrase = PREFIX_KEY + phrase
kvs[phrase][(lang, dictionary_name, canonical)] = is_canonical
kvs = [(k, '|'.join([l, d, str(int(i)), safe_encode(c)])) for k, vals in kvs.iteritems() for (l, d, c), i in vals.iteritems()]
self.trie = BytesTrie(kvs)
self.configured = True
def include_component_simple(self,
component,
containing_ids,
country=None):
containing = self.get_property(('components', component, 'containing'),
country=country,
default=None)
if containing is not None:
for c in containing:
if (c['type'], safe_encode(c['id'])) in containing_ids:
return random.random() < c['probability']
probability = self.get_property(
('components', component, 'probability'),
country=country,
default=0.0)
return random.random() < probability
def __init__(self, *dictionaries):
self.dictionaries = dictionaries
self.canonicals = {}
kvs = defaultdict(OrderedDict)
for language in address_phrase_dictionaries.languages:
for dictionary_name in self.dictionaries:
is_suffix_dictionary = 'suffixes' in dictionary_name
is_prefix_dictionary = 'prefixes' in dictionary_name
for phrases in address_phrase_dictionaries.phrases.get(
(language, dictionary_name), []):
canonical = phrases[0]
canonical_normalized = normalize_string(canonical)
self.canonicals[(canonical, language,
dictionary_name)] = phrases[1:]
for i, phrase in enumerate(phrases):
if phrase in POSSIBLE_ROMAN_NUMERALS:
continue
is_canonical = normalize_string(
phrase) == canonical_normalized
if is_suffix_dictionary:
phrase = SUFFIX_KEY + phrase[::-1]
elif is_prefix_dictionary:
phrase = PREFIX_KEY + phrase
kvs[phrase][(language, dictionary_name,
canonical)] = is_canonical
kvs = [(k, '|'.join([l, d, str(int(i)),
safe_encode(c)])) for k, vals in kvs.iteritems()
for (l, d, c), i in vals.iteritems()]
self.trie = BytesTrie(kvs)
def data_and_dependencies(self, path):
data = json.load(open(path))
props = data['properties']
_, filename = os.path.split(path)
current_wof_id = filename.rsplit('.geojson', 1)[0]
dependencies = {}
for hierarchy in props.get('wof:hierarchy', []):
for key, wof_id in six.iteritems(hierarchy):
wof_id = safe_encode(wof_id)
if wof_id in dependencies or wof_id == current_wof_id:
continue
if not self.client.exists_locally(wof_id):
continue
value = self.client.load(wof_id)
# Only include properties, not all the polygon data
dependencies[wof_id] = value.get('properties', {})
return data, dependencies
def data_and_dependencies(self, path):
data = json.load(open(path))
props = data['properties']
_, filename = os.path.split(path)
current_wof_id = filename.rsplit('.geojson', 1)[0]
dependencies = {}
for hierarchy in props.get('wof:hierarchy', []):
for key, wof_id in six.iteritems(hierarchy):
wof_id = safe_encode(wof_id)
if wof_id in dependencies or wof_id == current_wof_id:
continue
if not self.client.exists_locally(wof_id):
continue
value = self.client.load(wof_id)
# Only include properties, not all the polygon data
dependencies[wof_id] = value.get('properties', {})
return data, dependencies
def include_polygon(self, props):
return 'ISO3166-1:alpha2' in props or 'ISO3166-2' in props or (
props.get('type', 'relation'), safe_encode(props.get(
'id', ''))) in osm_admin1_ids
def parse_numex_rules(dirname=NUMEX_DATA_DIR, outfile=NUMEX_RULES_FILE):
all_keys = []
all_rules = []
all_ordinal_indicators = []
all_stopwords = []
all_languages = []
out = open(outfile, 'w')
for filename in os.listdir(dirname):
path = os.path.join(dirname, filename)
if not os.path.isfile(path) or not filename.endswith('.yaml'):
continue
language = filename.split('.yaml', 1)[0]
data = yaml.load(open(path))
whole_words_only = data.get('whole_words_only', False)
rules = data.get('rules', [])
rule_index = len(all_rules)
for rule in rules:
invalid_keys = set(rule.keys()) - valid_numex_keys
if invalid_keys:
raise InvalidNumexRuleException(u'Invalid keys: ({}) for language {}, rule: {}'.format(u','.join(invalid_keys), language, rule))
gender = gender_map[rule.get('gender')]
rule_type = rule_type_map[rule['type']]
key = rule['name']
value = rule['value']
radix = rule.get('radix', 10)
rule_category = rule.get('category')
category = category_map.get(rule_category)
if category is None:
continue
left_context_type = left_context_map[rule.get('left')]
right_context_type = right_context_map[rule.get('right')]
all_keys.append(unicode(numex_key_template.format(key=key)))
all_rules.append(unicode(numex_rule_template.format(
language=language,
rule_type=rule_type,
gender=gender,
category=category,
left_context_type=left_context_type,
right_context_type=right_context_type,
value=value,
radix=radix
)))
ordinal_indicator_index = len(all_ordinal_indicators)
ordinal_indicators = data.get('ordinal_indicators', [])
num_ordinal_indicators = 0
for rule in ordinal_indicators:
gender = gender_map[rule.get('gender')]
category = category_map[rule.get('category')]
invalid_ordinal_keys = set(rule.keys()) - valid_ordinal_keys
if invalid_ordinal_keys:
raise InvalidNumexRuleException(u'Invalid keys ({}) in ordinal rule for language {}, rule: {}'.format(u','.join(invalid_ordinal_keys), language, rule))
for key, suffixes in rule['suffixes'].iteritems():
for suffix in suffixes:
all_ordinal_indicators.append(unicode(ordinal_indicator_template.format(
key=key,
value=suffix,
gender=gender,
category=category
)))
num_ordinal_indicators += len(suffixes)
stopwords = data.get('stopwords', [])
stopword_index = len(all_stopwords)
num_stopwords = len(stopwords)
for stopword in stopwords:
all_keys.append(numex_key_template.format(key=unicode(stopword)))
all_rules.append(stopword_rule)
num_rules = len(rules) + len(stopwords)
all_languages.append(unicode(language_template.format(
language=language,
whole_words_only=int(whole_words_only),
rule_index=rule_index,
num_rules=num_rules,
ordinal_indicator_index=ordinal_indicator_index,
num_ordinal_indicators=num_ordinal_indicators
)))
out.write(safe_encode(numex_rules_data_template.format(
numex_keys=u''',
'''.join(all_keys),
numex_rules=u''',
'''.join(all_rules),
ordinal_indicator_rules=u''',
'''.join(all_ordinal_indicators),
stopwords=u''',
'''.join(all_stopwords),
languages=u''',
'''.join(all_languages),
)))
out.close()
def intersections(self):
'''
Generator which yields tuples like:
(node_id, lat, lon, {way_id: way_props})
'''
i = 0
node_ids = array.array('l')
node_counts = array.array('i')
for element_id, props, deps in parse_osm(self.filename,
dependencies=True):
props = {
safe_decode(k): safe_decode(v)
for k, v in six.iteritems(props)
}
if element_id.startswith('node'):
node_id = long(element_id.split(':')[-1])
node_ids.append(node_id)
node_counts.append(0)
self.node_props.Put(safe_encode(node_id), json.dumps(props))
elif element_id.startswith('way'):
# Don't care about the ordering of the nodes, and want uniques e.g. for circular roads
deps = set(deps)
# Get node indices by binary search
for node_id in deps:
try:
node_index = self.binary_search(node_ids, node_id)
except ValueError:
continue
if node_index is None:
continue
node_counts[node_index] += 1
if i % 1000 == 0 and i > 0:
self.logger.info('doing {}s, at {}'.format(
element_id.split(':')[0], i))
i += 1
for i, count in enumerate(node_counts):
if count > 1:
self.node_ids.append(node_ids[i])
del node_ids
del node_counts
i = 0
for element_id, props, deps in parse_osm(self.filename,
dependencies=True):
if element_id.startswith('node'):
node_id = long(element_id.split(':')[-1])
node_index = self.binary_search(self.node_ids, node_id)
elif element_id.startswith('way'):
props = {
safe_decode(k): safe_decode(v)
for k, v in six.iteritems(props)
}
way_id = long(element_id.split(':')[-1])
props['id'] = way_id
for node_id in deps:
node_index = self.binary_search(self.node_ids, node_id)
if node_index is not None:
self.intersection_edges_nodes.append(node_id)
self.intersection_edges_ways.append(way_id)
self.way_props.Put(safe_encode(way_id),
json.dumps(props))
if i % 1000 == 0 and i > 0:
self.logger.info('second pass, doing {}s, at {}'.format(
element_id.split(':')[0], i))
i += 1
i = 0
indices = numpy.argsort(self.intersection_edges_nodes)
self.intersection_edges_nodes = numpy.fromiter(
(self.intersection_edges_nodes[i] for i in indices),
dtype=numpy.uint64)
self.intersection_edges_ways = numpy.fromiter(
(self.intersection_edges_ways[i] for i in indices),
dtype=numpy.uint64)
del indices
idx = 0
# Need to make a copy here otherwise will change dictionary size during iteration
for node_id, g in groupby(self.intersection_edges_nodes):
group_len = sum((1 for j in g))
node_props = json.loads(self.node_props.Get(safe_encode(node_id)))
way_indices = self.intersection_edges_ways[idx:idx + group_len]
all_ways = [
json.loads(self.way_props.Get(safe_encode(w)))
for w in way_indices
]
way_names = set()
ways = []
for way in all_ways:
if way['name'] in way_names:
continue
ways.append(way)
way_names.add(way['name'])
idx += group_len
if i % 1000 == 0 and i > 0:
self.logger.info('checking intersections, did {}'.format(i))
i += 1
if len(ways) > 1:
node_index = self.binary_search(self.node_ids, node_id)
yield self.node_ids[node_index], node_props, ways
def __init__(self, config_file=BOUNDARY_NAMES_CONFIG):
config = yaml.load(open(config_file))
default_names = nested_get(config, ('names', 'keys'))
name_keys, probs = alternative_probabilities(default_names)
self.name_keys = name_keys
self.name_key_probs = cdf(probs)
self.component_name_keys = {}
for component, component_config in six.iteritems(nested_get(config, ('names', 'components'), default={})):
component_names = component_config.get('keys')
component_name_keys, component_probs = alternative_probabilities(component_names)
self.component_name_keys[component] = (component_name_keys, cdf(component_probs))
self.country_regex_replacements = defaultdict(list)
for props in nested_get(config, ('names', 'regex_replacements',), default=[]):
country = props.get('country')
re_flags = re.I | re.UNICODE
if not props.get('case_insensitive', True):
re.flags ^= re.I
pattern = re.compile(props['pattern'], re_flags)
replace_group = props['replace_with_group']
replace_probability = props['replace_probability']
self.country_regex_replacements[country].append((pattern, replace_group, replace_probability))
self.country_regex_replacements = dict(self.country_regex_replacements)
self.prefixes = {}
self.prefix_regexes = {}
self.suffixes = {}
self.suffix_regexes = {}
for language, components in six.iteritems(nested_get(config, ('names', 'prefixes', 'language'), default={}) ):
for component, affixes in six.iteritems(components):
affix_values, probs = alternative_probabilities(affixes)
for val in affix_values:
if 'prefix' not in val:
raise AssertionError(six.u('Invalid prefix value for (language={}, component={}): {} ').format(language, component, val))
prefix_regex = six.u('|').join([six.u('(?:{} )').format(self._string_as_regex(v['prefix'])) if v.get('whitespace') else self._string_as_regex(v['prefix']) for v in affix_values])
self.prefix_regexes[(language, component)] = re.compile(six.u('^{}').format(prefix_regex), re.I | re.U)
if not isclose(sum(probs), 1.0):
affix_values.append(None)
probs.append(1.0 - sum(probs))
affix_probs_cdf = cdf(probs)
self.prefixes[(language, component)] = affix_values, affix_probs_cdf
for language, components in six.iteritems(nested_get(config, ('names', 'suffixes', 'language'), default={}) ):
for component, affixes in six.iteritems(components):
affix_values, probs = alternative_probabilities(affixes)
for val in affix_values:
if 'suffix' not in val:
raise AssertionError(six.u('Invalid suffix value for (language={}, component={}): {} ').format(language, component, val))
suffix_regex = six.u('|').join([six.u('(?: {})').format(self._string_as_regex(v['suffix'])) if v.get('whitespace') else self._string_as_regex(v['suffix']) for v in affix_values])
self.suffix_regexes[(language, component)] = re.compile(six.u('{}$').format(suffix_regex), re.I | re.U)
if not isclose(sum(probs), 1.0):
affix_values.append(None)
probs.append(1.0 - sum(probs))
affix_probs_cdf = cdf(probs)
self.suffixes[(language, component)] = affix_values, affix_probs_cdf
self.exceptions = {}
for props in nested_get(config, ('names', 'exceptions'), default=[]):
object_type = props['type']
object_id = safe_encode(props['id'])
keys = [props['default']]
probs = [props['probability']]
for alt in props.get('alternatives', []):
keys.append(alt['alternative'])
probs.append(alt['probability'])
probs = cdf(probs)
self.exceptions[(object_type, object_id)] = (keys, probs)
def create_address_expansion_rules_file(base_dir=ADDRESS_EXPANSIONS_DIR,
output_file=ADDRESS_DATA_FILE,
header_file=ADDRESS_HEADER_FILE):
address_languages = []
expansion_rules = []
canonical_strings = []
max_dictionary_types = 0
for language in address_phrase_dictionaries.languages:
num_language_rules = 0
language_index = len(expansion_rules)
language_canonical_dictionaries = defaultdict(list)
canonical_indices = {}
for dictionary_name in address_phrase_dictionaries.language_dictionaries[
language]:
dictionary_type = gazetteer_types[dictionary_name]
for phrases in address_phrase_dictionaries.phrases[(
language, dictionary_name)]:
canonical = phrases[0]
if len(phrases) > 1:
canonical_index = canonical_indices.get(canonical, None)
if canonical_index is None:
canonical_index = len(canonical_strings)
canonical_strings.append(quote_string(canonical))
canonical_indices[canonical] = canonical_index
else:
canonical_index = -1
for i, p in enumerate(phrases):
language_canonical_dictionaries[
p, canonical_index if i > 0 else -1].append(
dictionary_type)
for (
phrase, canonical_index
), dictionary_types in language_canonical_dictionaries.iteritems():
max_dictionary_types = max(max_dictionary_types,
len(dictionary_types))
rule_template = address_expansion_rule_template.format(
phrase=quote_string(phrase),
num_dictionaries=str(len(dictionary_types)),
dictionaries=', '.join(dictionary_types),
canonical_index=canonical_index)
expansion_rules.append(rule_template)
num_language_rules += 1
address_languages.append(
address_language_index_template.format(
language=quote_string(language),
index=language_index,
length=num_language_rules))
header = address_expansion_rule_header_template.format(
max_dictionary_types=str(max_dictionary_types))
out = open(header_file, 'w')
out.write(safe_encode(header))
out.close()
data_file = address_expansion_data_file_template.format(
canonical_strings=u''',
'''.join(canonical_strings),
expansion_rules=u''',
'''.join(expansion_rules),
address_languages=u''',
'''.join(address_languages),
)
out = open(output_file, 'w')
out.write(safe_encode(data_file))
out.close()
def tokenize(s):
u = safe_decode(s)
s = safe_encode(s)
return [(safe_decode(s[start:start + length]), token_types.from_id(token_type))
for start, length, token_type in _tokenize.tokenize(u)]
def parse_numex_rules(dirname=NUMEX_DATA_DIR, outfile=NUMEX_RULES_FILE):
all_keys = []
all_rules = []
all_ordinal_indicators = []
all_stopwords = []
all_languages = []
out = open(outfile, 'w')
for filename in os.listdir(dirname):
path = os.path.join(dirname, filename)
if not os.path.isfile(path) or not filename.endswith('.json'):
continue
language = filename.split('.json', 1)[0]
data = json.load(open(path))
whole_words_only = data.get('whole_words_only', False)
rules = data.get('rules', [])
rule_index = len(all_rules)
for rule in rules:
invalid_keys = set(rule.keys()) - valid_numex_keys
if invalid_keys:
raise InvalidNumexRuleException(
u'Invalid keys: ({}) for language {}, rule: {}'.format(
u','.join(invalid_keys), language, rule))
gender = gender_map[rule.get('gender')]
rule_type = rule_type_map[rule['type']]
key = rule['name']
value = rule['value']
radix = rule.get('radix', 10)
category = category_map[rule.get('category')]
left_context_type = left_context_map[rule.get('left')]
right_context_type = right_context_map[rule.get('right')]
all_keys.append(unicode(numex_key_template.format(key=key)))
all_rules.append(
unicode(
numex_rule_template.format(
language=language,
rule_type=rule_type,
gender=gender,
category=category,
left_context_type=left_context_type,
right_context_type=right_context_type,
value=value,
radix=radix)))
ordinal_indicator_index = len(all_ordinal_indicators)
ordinal_indicators = data.get('ordinal_indicators', [])
num_ordinal_indicators = 0
for rule in ordinal_indicators:
gender = gender_map[rule.get('gender')]
category = category_map[rule.get('category')]
invalid_ordinal_keys = set(rule.keys()) - valid_ordinal_keys
if invalid_ordinal_keys:
raise InvalidNumexRuleException(
u'Invalid keys ({}) in ordinal rule for language {}, rule: {}'
.format(u','.join(invalid_ordinal_keys), language, rule))
for key, suffixes in rule['suffixes'].iteritems():
for suffix in suffixes:
all_ordinal_indicators.append(
unicode(
ordinal_indicator_template.format(
key=key,
value=suffix,
gender=gender,
category=category)))
num_ordinal_indicators += len(suffixes)
stopwords = data.get('stopwords', [])
stopword_index = len(all_stopwords)
num_stopwords = len(stopwords)
for stopword in stopwords:
all_keys.append(numex_key_template.format(key=unicode(stopword)))
all_rules.append(stopword_rule)
num_rules = len(rules) + len(stopwords)
all_languages.append(
unicode(
language_template.format(
language=language,
whole_words_only=int(whole_words_only),
rule_index=rule_index,
num_rules=num_rules,
ordinal_indicator_index=ordinal_indicator_index,
num_ordinal_indicators=num_ordinal_indicators)))
out.write(
safe_encode(
numex_rules_data_template.format(
numex_keys=u''',
'''.join(all_keys),
numex_rules=u''',
'''.join(all_rules),
ordinal_indicator_rules=u''',
'''.join(all_ordinal_indicators),
stopwords=u''',
'''.join(all_stopwords),
languages=u''',
'''.join(all_languages),
)))
out.close()
def encode_field(value):
return multispace_regex.sub(' ', safe_encode((value if value is not None else '')))
def abbreviate(gazetteer, s, language, abbreviate_prob=0.3, separate_prob=0.2, add_period_hyphen_prob=0.3):
'''
Abbreviations
-------------
OSM discourages abbreviations, but to make our training data map better
to real-world input, we can safely replace the canonical phrase with an
abbreviated version and retain the meaning of the words
'''
raw_tokens = tokenize_raw(s)
s_utf8 = safe_encode(s)
tokens = [(safe_decode(s_utf8[o:o + l]), token_types.from_id(c)) for o, l, c in raw_tokens]
norm_tokens = [(t.lower() if c in token_types.WORD_TOKEN_TYPES else t, c) for t, c in tokens]
n = len(tokens)
abbreviated = []
i = 0
def abbreviated_tokens(i, tokens, t, c, length, data, space_token=six.u(' ')):
data = [d.split(six.b('|')) for d in data]
# local copy
abbreviated = []
n = len(t)
# Append the original tokens with whitespace if there is any
if random.random() > abbreviate_prob or not any((int(is_canonical) and lang in (language, 'all') for lang, dictionary, is_canonical, canonical in data)):
for j, (t_i, c_i) in enumerate(t):
abbreviated.append(tokens[i + j][0])
if j < n - 1:
abbreviated.append(space_token)
return abbreviated
for lang, dictionary, is_canonical, canonical in data:
if lang not in (language, 'all'):
continue
is_canonical = int(is_canonical)
is_stopword = dictionary == 'stopword'
is_prefix = dictionary.startswith('concatenated_prefixes')
is_suffix = dictionary.startswith('concatenated_suffixes')
is_separable = is_prefix or is_suffix and dictionary.endswith('_separable') and len(t[0][0]) > length
suffix = None
prefix = None
if not is_canonical:
continue
if not is_prefix and not is_suffix:
abbreviations = gazetteer.canonicals.get((canonical, lang, dictionary))
# TODO: maybe make this a Zipfian choice e.g. so "St" gets chosen most often for "Street"
# would require an audit of the dictionaries though so abbreviations are listed from
# left-to-right by frequency of usage
token = random.choice(abbreviations) if abbreviations else canonical
token = recase_abbreviation(token, tokens[i:i + len(t)], space_token=space_token)
abbreviated.append(token)
break
elif is_prefix:
token = tokens[i][0]
prefix, token = token[:length], token[length:]
abbreviated.append(prefix)
if random.random() < separate_prob:
sub_tokens = tokenize(token)
if sub_tokens and sub_tokens[0][1] in (token_types.HYPHEN, token_types.DASH):
token = six.u('').join((t for t, c in sub_tokens[1:]))
abbreviated.append(space_token)
if token.islower():
abbreviated.append(token.title())
else:
abbreviated.append(token)
abbreviated.append(space_token)
break
elif is_suffix:
token = tokens[i][0]
token, suffix = token[:-length], token[-length:]
concatenated_abbreviations = gazetteer.canonicals.get((canonical, lang, dictionary), [])
separated_abbreviations = []
phrase = gazetteer.trie.get(suffix.rstrip('.'))
suffix_data = [safe_decode(d).split(six.u('|')) for d in (phrase or [])]
for l, d, _, c in suffix_data:
if l == lang and c == canonical:
separated_abbreviations.extend(gazetteer.canonicals.get((canonical, lang, d)))
separate = random.random() < separate_prob
if concatenated_abbreviations and not separate:
abbreviation = random.choice(concatenated_abbreviations)
elif separated_abbreviations:
abbreviation = random.choice(separated_abbreviations)
else:
abbreviation = canonical
if separate:
sub_tokens = tokenize(token)
if sub_tokens and sub_tokens[-1][1] in (token_types.HYPHEN, token_types.DASH):
token = six.u('').join((t for t, c in sub_tokens[:-1]))
abbreviated.append(token)
if separate:
abbreviated.append(space_token)
if suffix.isupper():
abbreviated.append(abbreviation.upper())
elif separate:
abbreviated.append(abbreviation.title())
else:
abbreviated.append(abbreviation)
break
else:
for j, (t_i, c_i) in enumerate(t):
abbreviated.append(tokens[i + j][0])
if j < n - 1:
abbreviated.append(space_token)
return abbreviated
for t, c, length, data in gazetteer.filter(norm_tokens):
if c == token_types.PHRASE:
abbrev_tokens = abbreviated_tokens(i, tokens, t, c, length, data)
abbreviated.extend(abbrev_tokens)
if i + len(t) < n and raw_tokens[i + len(t)][0] > sum(raw_tokens[i + len(t) - 1][:2]):
abbreviated.append(six.u(' '))
i += len(t)
else:
token = tokens[i][0]
if not non_breaking_dash_regex.search(token):
abbreviated.append(token)
else:
sub_tokens = tokenize(non_breaking_dash_regex.sub(six.u(' '), token))
sub_tokens_norm = [(t.lower() if c in token_types.WORD_TOKEN_TYPES else t, c) for t, c in sub_tokens]
sub_token_abbreviated = []
sub_i = 0
sub_n = len(sub_tokens)
for t, c, length, data in gazetteer.filter(sub_tokens_norm):
if c == token_types.PHRASE:
abbrev_tokens = abbreviated_tokens(sub_i, sub_tokens, t, c, length, data, space_token=six.u('-'))
sub_token_abbreviated.extend(abbrev_tokens)
sub_i += len(t)
if sub_i < sub_n:
if abbrev_tokens and random.random() < add_period_hyphen_prob and not abbrev_tokens[-1].endswith(six.u('.')) and not abbrev_tokens[-1].lower().endswith(sub_tokens_norm[sub_i - 1][0]):
sub_token_abbreviated.append(six.u('.'))
sub_token_abbreviated.append(six.u('-'))
else:
sub_token_abbreviated.append(sub_tokens[sub_i][0])
sub_i += 1
if sub_i < sub_n:
sub_token_abbreviated.append(six.u('-'))
abbreviated.append(six.u('').join(sub_token_abbreviated))
if i < n - 1 and raw_tokens[i + 1][0] > sum(raw_tokens[i][:2]):
abbreviated.append(six.u(' '))
i += 1
return six.u('').join(abbreviated).strip()
class OpenAddressesFormatter(object):
field_regex_replacements = {
# All fields
None: [
(re.compile('<\s*null\s*>', re.I), u''),
(re.compile('[\s]{2,}'), six.u(' ')),
(re.compile('\`'), u"'"),
(re.compile('\-?\*'), u""),
],
AddressFormatter.HOUSE_NUMBER: [
# Most of the house numbers in Montreal start with "#"
(re.compile('^#', re.UNICODE), u''),
# Some house numbers have multiple hyphens
(re.compile('[\-]{2,}'), u'-'),
# Some house number ranges are split up like "12 -14"
(re.compile('[\s]*\-[\s]*'), u'-'),
]
}
unit_type_regexes = {}
for (lang, dictionary_type), values in six.iteritems(
address_phrase_dictionaries.phrases):
if dictionary_type == 'unit_types_numbered':
unit_phrases = [
safe_encode(p) for p in itertools.chain(*values) if len(p) > 2
]
pattern = re.compile(
r'\b(?:{})\s+(?:#?\s*)(?:[\d]+|[a-z]|[a-z]\-?[\d]+|[\d]+\-?[a-z])\s*$'
.format(safe_encode('|').join(unit_phrases)),
re.I | re.UNICODE)
unit_type_regexes[lang] = pattern
def __init__(self, components, country_rtree, debug=False):
self.components = components
self.country_rtree = country_rtree
self.debug = debug
self.formatter = AddressFormatter()
class validators:
@classmethod
def validate_postcode(cls, postcode):
'''
Postcodes that are all zeros are improperly-formatted NULL values
'''
return not all((c in ('0', '-', '.', ' ', ',') for c in postcode))
@classmethod
def validate_street(cls, street):
'''
Streets should not be simple numbers. If they are it's probably a
copy/paste error and should be the house number.
'''
return not is_numeric(street)
@classmethod
def validate_house_number(cls, house_number):
'''
House number doesn't necessarily have to be numeric, but in some of the
OpenAddresses data sets the house number field is equal to the capitalized
street name, so this at least provides protection against insane values
for house number at the cost of maybe missing a few houses numbered "A", etc.
Also OpenAddresses primarily comes from county GIS servers, etc. which use
a variety of database schemas and don't always handle NULLs very well. Again,
while a single zero is a valid house number, in OpenAddresses it's more likely
an error
While a single zero is a valid house number, more than one zero is not, or
at least not in OpenAddresses
'''
try:
house_number = int(house_number.strip())
return house_number > 0
except (ValueError, TypeError):
house_number = house_number.strip()
return house_number and (
is_numeric(house_number)
or fraction_regex.match(house_number)
or number_space_letter_regex.match(house_number)
or number_slash_number_regex.match(house_number)
or number_fraction_regex.match(house_number)) and not all(
(c == '0' for c in house_number if c.isdigit()))
@classmethod
def validate_house_number_sin_numero(cls, house_number):
if sin_numero_regex.match(house_number):
return True
return cls.validate_house_number(house_number)
@classmethod
def validate_russian_house_number(cls, house_number):
if dom_korpus_stroyeniye_regex.match(house_number):
return True
elif uchastok_regex.match(house_number):
return True
elif bea_nomera_regex.match(house_number):
return True
return cls.validate_house_number(house_number)
@classmethod
def validate_colombian_house_number(cls, house_number):
return True
@classmethod
def validate_chinese_house_number(cls, house_number):
if not house_number:
return False
tokens = tokenize(house_number)
if all((c in token_types.NUMERIC_TOKEN_TYPES or t in (u'号', u'栋',
u'附'))
for t, c in tokens):
return True
return cls.validate_house_number(house_number)
component_validators = {
AddressFormatter.HOUSE_NUMBER: validators.validate_house_number,
AddressFormatter.ROAD: validators.validate_street,
AddressFormatter.POSTCODE: validators.validate_postcode,
}
language_validators = {
SPANISH: {
AddressFormatter.HOUSE_NUMBER:
validators.validate_house_number_sin_numero,
},
PORTUGUESE: {
AddressFormatter.HOUSE_NUMBER:
validators.validate_house_number_sin_numero,
},
RUSSIAN: {
AddressFormatter.HOUSE_NUMBER:
validators.validate_russian_house_number,
},
CHINESE: {
AddressFormatter.HOUSE_NUMBER:
validators.validate_chinese_house_number,
}
}
country_validators = {
Countries.COLOMBIA: {
AddressFormatter.HOUSE_NUMBER:
validators.validate_colombian_house_number
}
}
chinese_annex_regex = re.compile(u'([\d]+)(?![\d号栋])', re.U)
@classmethod
def format_chinese_house_number(cls, house_number):
if not house_number:
return house_number
return cls.chinese_annex_regex.sub(u'\\1号', house_number)
@classmethod
def format_colombian_house_number(cls, house_number):
house_number = house_number.strip()
match = colombian_standard_house_number_regex.match(house_number)
if match:
separator = random.choice((u'-', u' - ', u' '))
cross_street, building_number = match.groups()
numbers = []
if cross_street and u' ' in cross_street and random.choice(
(True, False)):
cross_street = cross_street.replace(u' ', u'')
if cross_street:
numbers.append(cross_street)
if building_number and u' ' in building_number and random.choice(
(True, False)):
building_number = building_number.replace(u' ', u'')
if building_number:
numbers.append(building_number)
if numbers:
house_number = separator.join(numbers)
house_number_prefixes = (u'#', u'no.', u'no', u'nº')
if random.choice((True, False)) and not any(
(house_number.lower().startswith(p)
for p in house_number_prefixes)):
house_number = u' '.join(
[random.choice(house_number_prefixes), house_number])
return house_number
def get_property(self, key, *configs):
for config in configs:
value = config.get(key, None)
if value is not None:
return value
return None
def cldr_country_name(self, country_code, language, configs):
cldr_country_prob = float(
self.get_property('cldr_country_probability', *configs))
country_name = None
if random.random() < cldr_country_prob:
localized, iso_3166, alpha2, alpha3 = values = range(4)
localized_prob = float(
self.get_property('localized_name_probability', *configs))
iso_3166_prob = float(
self.get_property('iso_3166_name_probability', *configs))
alpha2_prob = float(
self.get_property('iso_alpha_2_code_probability', *configs))
alpha3_prob = float(
self.get_property('iso_alpha_3_code_probability', *configs))
probs = cdf(
[localized_prob, iso_3166_prob, alpha2_prob, alpha3_prob])
country_type = weighted_choice(values, probs)
country_name = country_code.upper()
if country_type == localized:
country_name = country_names.localized_name(
country_code, language) or country_names.localized_name(
country_code) or country_name
elif country_type == iso_3166:
country_name = country_names.iso3166_name(country_code)
elif country_type == alpha3:
country_name = country_names.alpha3_code(
country_code) or country_name
return country_name
@classmethod
def cleanup_number(cls, num, strip_commas=False):
num = num.strip()
if strip_commas:
num = num.replace(six.u(','), six.u(''))
try:
num_int = int(num)
except (ValueError, TypeError):
try:
num_float = float(num)
leading_zeros = 0
for c in num:
if c == six.u('0'):
leading_zeros += 1
else:
break
num = safe_decode(int(num_float))
if leading_zeros:
num = six.u('{}{}').format(six.u('0') * leading_zeros, num)
except (ValueError, TypeError):
pass
return num
@classmethod
def fix_component_encodings(cls, components):
return {
k: ftfy.fix_encoding(safe_decode(v))
for k, v in six.iteritems(components)
}
def formatted_addresses(self,
country_dir,
path,
configs,
tag_components=True):
abbreviate_street_prob = float(
self.get_property('abbreviate_street_probability', *configs))
separate_street_prob = float(
self.get_property('separate_street_probability', *configs) or 0.0)
abbreviate_unit_prob = float(
self.get_property('abbreviate_unit_probability', *configs))
separate_unit_prob = float(
self.get_property('separate_unit_probability', *configs) or 0.0)
abbreviate_toponym_prob = float(
self.get_property('abbreviate_toponym_probability', *configs))
add_osm_boundaries = bool(
self.get_property('add_osm_boundaries', *configs) or False)
add_osm_neighborhoods = bool(
self.get_property('add_osm_neighborhoods', *configs) or False)
osm_neighborhood_overrides_city = self.get_property(
'osm_neighborhood_overrides_city', *configs)
non_numeric_units = bool(
self.get_property('non_numeric_units', *configs) or False)
house_number_strip_commas = bool(
self.get_property('house_number_strip_commas', *configs) or False)
numeric_postcodes_only = bool(
self.get_property('numeric_postcodes_only', *configs) or False)
postcode_strip_non_digit_chars = bool(
self.get_property('postcode_strip_non_digit_chars', *configs)
or False)
address_only_probability = float(
self.get_property('address_only_probability', *configs))
place_only_probability = float(
self.get_property('place_only_probability', *configs))
place_and_postcode_probability = float(
self.get_property('place_and_postcode_probability', *configs))
city_replacements = self.get_property('city_replacements', *configs)
override_country_dir = self.get_property('override_country_dir',
*configs)
postcode_length = int(
self.get_property('postcode_length', *configs) or 0)
drop_address_probability = place_only_probability + place_and_postcode_probability
ignore_rows_missing_fields = set(
self.get_property('ignore_rows_missing_fields', *configs) or [])
ignore_fields_containing = {
field: re.compile(
six.u('|').join(
[six.u('(?:{})').format(safe_decode(v)) for v in value]),
re.I | re.UNICODE)
for field, value in six.iteritems(
dict(
self.get_property('ignore_fields_containing', *configs)
or {}))
}
alias_fields_containing = {
field:
[(re.compile(v['pattern'], re.I | re.UNICODE), v) for v in value]
for field, value in six.iteritems(
dict(
self.get_property('alias_fields_containing', *configs)
or {}))
}
config_language = self.get_property('language', *configs)
add_components = self.get_property('add', *configs)
fields = self.get_property('fields', *configs)
if not fields:
return
field_map = {
field_name: f['component']
for field_name, f in six.iteritems(fields)
}
mapped_values = {
f['component']: f['value_map']
for f in six.itervalues(fields)
if hasattr(f.get('value_map'), 'get')
}
f = open(path)
reader = unicode_csv_reader(f)
headers = reader.next()
header_indices = {
i: field_map[k]
for i, k in enumerate(headers) if k in field_map
}
latitude_index = headers.index('LAT')
longitude_index = headers.index('LON')
# Clear cached polygons
self.components.osm_admin_rtree.clear_cache()
self.components.neighborhoods_rtree.clear_cache()
for row in reader:
try:
latitude = float(row[latitude_index])
longitude = float(row[longitude_index])
except (ValueError, TypeError):
continue
language = config_language
components = {}
skip_record = False
for i, key in six.iteritems(header_indices):
value = row[i].strip()
if not value and key in ignore_rows_missing_fields:
skip_record = True
break
elif not value:
continue
if key in mapped_values:
value = mapped_values[key].get(value, value)
if key == AddressFormatter.ROAD and language == SPANISH:
value = self.components.spanish_street_name(value)
if key == AddressFormatter.POSTCODE:
value = self.cleanup_number(value)
if postcode_strip_non_digit_chars:
value = six.u('').join(
(c for c in value if c.isdigit()))
if value and not is_numeric(
value) and numeric_postcodes_only:
continue
else:
if postcode_length:
value = value.zfill(
postcode_length)[:postcode_length]
if key in AddressFormatter.BOUNDARY_COMPONENTS and key != AddressFormatter.POSTCODE:
if add_osm_boundaries:
continue
value = self.components.cleaned_name(
value, first_comma_delimited_phrase=True)
if value and ((len(value) < 2
and not get_string_script(value)[0].lower()
in ideographic_scripts)
or is_numeric(value)):
continue
if not_applicable_regex.match(value) or null_regex.match(
value) or unknown_regex.match(value):
continue
for exp, sub_val in self.field_regex_replacements.get(key, []):
value = exp.sub(sub_val, value)
for exp, sub_val in self.field_regex_replacements.get(
None, []):
value = exp.sub(sub_val, value)
value = value.strip(', -')
validator = self.country_validators.get(country_dir, {}).get(
key,
self.language_validators.get(language, {}).get(
key, self.component_validators.get(key, None)))
if validator is not None and not validator(value):
continue
if key in ignore_fields_containing and ignore_fields_containing[
key].search(value):
continue
for (pattern, alias) in alias_fields_containing.get(key, []):
if pattern.search(value):
if 'component' in alias:
key = alias['component']
if value:
components[key] = value
if skip_record:
continue
if components:
country, candidate_languages = self.country_rtree.country_and_languages(
latitude, longitude)
if not (country and candidate_languages) or (
country != country_dir and not override_country_dir):
country = country_dir
candidate_languages = get_country_languages(country)
if not candidate_languages:
continue
candidate_languages = candidate_languages.items()
components = self.fix_component_encodings(components)
if language is None:
language = AddressComponents.address_language(
components, candidate_languages)
street = components.get(AddressFormatter.ROAD, None)
if street is not None:
street = street.strip()
street = AddressComponents.cleaned_name(street)
if language == UNKNOWN_LANGUAGE:
strip_unit_language = candidate_languages[0][
0] if candidate_languages else None
else:
strip_unit_language = language
street = self.components.strip_unit_phrases_for_language(
street, strip_unit_language)
street = abbreviate(street_types_gazetteer,
street,
language,
abbreviate_prob=abbreviate_street_prob,
separate_prob=separate_street_prob)
components[AddressFormatter.ROAD] = street
house_number = components.get(AddressFormatter.HOUSE_NUMBER,
None)
if house_number:
house_number = self.cleanup_number(
house_number, strip_commas=house_number_strip_commas)
if language == CHINESE:
house_number = self.format_chinese_house_number(
house_number)
if country_dir == Countries.COLOMBIA:
house_number = self.format_colombian_house_number(
house_number)
if house_number is not None:
components[
AddressFormatter.HOUSE_NUMBER] = house_number
unit = components.get(AddressFormatter.UNIT, None)
street_required = country not in (
Countries.JAPAN, Countries.CZECH_REPUBLIC
) and country not in Countries.FORMER_SOVIET_UNION_COUNTRIES
postcode = components.get(AddressFormatter.POSTCODE, None)
if postcode:
components[AddressFormatter.
POSTCODE] = PostalCodes.add_country_code(
postcode, country)
# If there's a postcode, we can still use just the city/state/postcode, otherwise discard
if (not street and street_required) or (
street and house_number and
(street.lower() == house_number.lower())) or (
unit and street and street.lower() == unit.lower()):
if not postcode:
continue
components = self.components.drop_address(components)
# Now that checks, etc. are completed, fetch unit and add phrases, abbreviate, etc.
unit = components.get(AddressFormatter.UNIT, None)
if unit is not None:
if is_numeric_strict(unit):
unit = Unit.phrase(unit, language, country=country)
elif non_numeric_units:
unit = abbreviate(unit_types_gazetteer,
unit,
language,
abbreviate_prob=abbreviate_unit_prob,
separate_prob=separate_unit_prob)
else:
unit = None
if unit is not None:
components[AddressFormatter.UNIT] = unit
else:
components.pop(AddressFormatter.UNIT)
unit = None
# CLDR country name
country_name = self.cldr_country_name(country, language,
configs)
if country_name:
components[AddressFormatter.COUNTRY] = country_name
for component_key in AddressFormatter.BOUNDARY_COMPONENTS:
component = components.get(component_key, None)
if component is not None:
component = abbreviate(
toponym_abbreviations_gazetteer,
component,
language,
abbreviate_prob=abbreviate_toponym_prob)
component = self.components.name_hyphens(component)
components[component_key] = component
# Any components specified to be added by the config (usually state)
if add_components:
for k, v in six.iteritems(add_components):
if k not in components:
components[k] = v
# Get named states occasionally, added component is usually a state code
address_state = self.components.state_name(
components, country, language)
if address_state:
components[AddressFormatter.STATE] = address_state
state = components.get(AddressFormatter.STATE)
if state:
state = self.components.abbreviated_state(
state, country, language)
if state:
components[AddressFormatter.STATE] = state
# This is expensive, so only turn on for files that don't supply their own city names
# or for which those names are flawed
osm_components = []
# Using population=0 instead of None means if there's no known population or
# we don't need to add OSM components, we assume the population of the town is
# very small and the place name shouldn't be used unqualified (i.e. needs information
# like state name to disambiguate it)
population = 0
unambiguous_city = False
if add_osm_boundaries or AddressFormatter.CITY not in components:
osm_components = self.components.osm_reverse_geocoded_components(
latitude, longitude)
self.components.add_admin_boundaries(
components, osm_components, country, language,
latitude, longitude)
categorized = self.components.categorized_osm_components(
country, osm_components)
for component, label in categorized:
if label == AddressFormatter.CITY:
unambiguous_city = self.components.unambiguous_wikipedia(
component, language)
if 'population' in component:
population = component['population']
break
if AddressFormatter.CITY not in components and city_replacements:
components.update({
k: v
for k, v in six.iteritems(city_replacements)
if k not in components
})
# The neighborhood index is cheaper so can turn on for whole countries
neighborhood_components = []
if add_osm_neighborhoods:
neighborhood_components = self.components.neighborhood_components(
latitude, longitude)
self.components.add_neighborhoods(
components,
neighborhood_components,
country,
language,
replace_city=osm_neighborhood_overrides_city)
self.components.cleanup_boundary_names(components)
self.components.country_specific_cleanup(components, country)
self.components.replace_name_affixes(components,
language,
country=country)
self.components.replace_names(components)
self.components.prune_duplicate_names(components)
self.components.remove_numeric_boundary_names(components)
self.components.add_house_number_phrase(components,
language,
country=country)
self.components.add_postcode_phrase(components,
language,
country=country)
# Component dropout
all_osm_components = osm_components + neighborhood_components
components = place_config.dropout_components(
components,
all_osm_components,
country=country,
population=population,
unambiguous_city=unambiguous_city)
self.components.add_genitives(components, language)
formatted = self.formatter.format_address(
components,
country,
language=language,
minimal_only=False,
tag_components=tag_components)
yield (language, country, formatted)
if random.random() < address_only_probability and street:
address_only_components = self.components.drop_places(
components)
address_only_components = self.components.drop_postcode(
address_only_components)
formatted = self.formatter.format_address(
address_only_components,
country,
language=language,
minimal_only=False,
tag_components=tag_components)
yield (language, country, formatted)
rand_val = random.random()
if street and house_number and rand_val < drop_address_probability:
components = self.components.drop_address(components)
if rand_val < place_and_postcode_probability:
components = self.components.drop_postcode(components)
if components and (len(components) > 1
or add_osm_boundaries):
formatted = self.formatter.format_address(
components,
country,
language=language,
minimal_only=False,
tag_components=tag_components)
yield (language, country, formatted)
def build_training_data(self,
base_dir,
out_dir,
tag_components=True,
sources_only=None):
all_sources_valid = sources_only is None
valid_sources = set()
if not all_sources_valid:
for source in sources_only:
if source.startswith(base_dir):
source = os.path.relpath(source, base_dir)
parts = source.strip('/ ').split('/')
if len(parts) > 3:
raise AssertionError(
'Sources may only have at maximum 3 parts')
valid_sources.add(tuple(parts))
if tag_components:
formatted_tagged_file = open(
os.path.join(out_dir,
OPENADDRESSES_FORMAT_DATA_TAGGED_FILENAME), 'w')
writer = csv.writer(formatted_tagged_file, 'tsv_no_quote')
else:
formatted_tagged_file = open(
os.path.join(out_dir, OPENADDRESSES_FORMAT_DATA_FILENAME), 'w')
writer = csv.writer(formatted_tagged_file, 'tsv_no_quote')
i = 0
for country_dir in sorted(openaddresses_config.country_configs.keys()):
country_config = openaddresses_config.country_configs[country_dir]
# Clear country cache for each new country
self.country_rtree.clear_cache()
for file_config in country_config.get('files', []):
filename = file_config['filename']
if not all_sources_valid and not (
(country_dir, filename) in valid_sources or
(country_dir, ) in valid_sources):
continue
print(six.u('doing {}/{}').format(country_dir, filename))
path = os.path.join(base_dir, country_dir, filename)
configs = (file_config, country_config,
openaddresses_config.config)
for language, country, formatted_address in self.formatted_addresses(
country_dir, path, configs,
tag_components=tag_components):
if not formatted_address or not formatted_address.strip():
continue
formatted_address = tsv_string(formatted_address)
if not formatted_address or not formatted_address.strip():
continue
if tag_components:
row = (language, country, formatted_address)
else:
row = (formatted_address, )
writer.writerow(row)
i += 1
if i % 1000 == 0 and i > 0:
print('did {} formatted addresses'.format(i))
if self.debug:
break
for subdir in sorted(country_config.get('subdirs', {}).keys()):
subdir_config = country_config['subdirs'][subdir]
subdir = safe_decode(subdir)
for file_config in subdir_config.get('files', []):
filename = file_config['filename']
if not all_sources_valid and not (
(country_dir, subdir, filename) in valid_sources or
(country_dir, subdir) in valid_sources or
(country_dir, ) in valid_sources):
continue
print(
six.u('doing {}/{}/{}').format(country_dir, subdir,
filename))
path = os.path.join(base_dir, country_dir, subdir,
filename)
configs = (file_config, subdir_config, country_config,
openaddresses_config.config)
for language, country, formatted_address in self.formatted_addresses(
country_dir,
path,
configs,
tag_components=tag_components):
if not formatted_address or not formatted_address.strip(
):
continue
formatted_address = tsv_string(formatted_address)
if not formatted_address or not formatted_address.strip(
):
continue
if tag_components:
row = (language, country, formatted_address)
else:
row = (formatted_address, )
writer.writerow(row)
i += 1
if i % 1000 == 0 and i > 0:
print('did {} formatted addresses'.format(i))
if self.debug:
break
def dropout_components(self,
components,
boundaries=(),
country=None,
population=None,
unambiguous_city=False):
containing_ids = set()
for boundary in boundaries:
object_type = boundary.get('type')
object_id = safe_encode(boundary.get('id', ''))
if not (object_type and object_id):
continue
containing_ids.add((object_type, object_id))
original_bitset = ComponentDependencies.component_bitset(components)
names = defaultdict(list)
admin_components = [
c for c in components if c in self.ADMIN_COMPONENTS
]
for c in admin_components:
names[components[c]].append(c)
same_name = set()
for c, v in six.iteritems(names):
if len(v) > 1:
same_name |= set(v)
new_components = components.copy()
city_replacements = set()
if AddressFormatter.CITY not in components:
city_replacements = self.city_replacements(country)
for component in admin_components:
include = self.include_component(component,
containing_ids,
country=country,
population=population,
unambiguous_city=unambiguous_city)
if not include and component not in city_replacements:
# Note: this check is for cities that have the same name as their admin
# areas e.g. Luxembourg, Luxembourg. In cases like this, if we were to drop
# city, we don't want to include country on its own. This should help the parser
# default to the city in ambiguous cases where only one component is specified.
if not (component == AddressFormatter.CITY
and component in same_name):
new_components.pop(component, None)
else:
value = components[component]
for c in names[value]:
new_components.pop(c, None)
for component in self.ADMIN_COMPONENTS:
value = self.get_property(('components', component, 'value'),
country=country,
default=None)
if not value:
values, probs = self.cdf_cache.get((country, component),
(None, None))
if values is None:
values = self.get_property(
('components', component, 'values'),
country=country,
default=None)
if values is not None:
values, probs = zip(*[(v['value'],
float(v['probability']))
for v in values])
probs = cdf(probs)
self.cdf_cache[(country, component)] = (values, probs)
if values is not None:
value = weighted_choice(values, probs)
if value is not None and component not in components and self.include_component(
component,
containing_ids,
country=country,
population=population,
unambiguous_city=unambiguous_city):
new_components[component] = value
self.drop_invalid_components(new_components,
country,
original_bitset=original_bitset)
return new_components
def create_address_expansion_rules_file(base_dir=ADDRESS_EXPANSIONS_DIR, output_file=ADDRESS_DATA_FILE, header_file=ADDRESS_HEADER_FILE):
address_languages = []
expansion_rules = []
canonical_strings = []
max_dictionary_types = 0
for language in address_phrase_dictionaries.languages:
num_language_rules = 0
language_index = len(expansion_rules)
language_canonical_dictionaries = defaultdict(list)
canonical_indices = {}
for dictionary_name in address_phrase_dictionaries.language_dictionaries[language]:
dictionary_type = gazetteer_types[dictionary_name]
for phrases in address_phrase_dictionaries.phrases[(language, dictionary_name)]:
canonical = phrases[0]
if len(phrases) > 1:
canonical_index = canonical_indices.get(canonical, None)
if canonical_index is None:
canonical_index = len(canonical_strings)
canonical_strings.append(quote_string(canonical))
canonical_indices[canonical] = canonical_index
else:
canonical_index = -1
for i, p in enumerate(phrases):
language_canonical_dictionaries[p, canonical_index if i > 0 else -1].append(dictionary_type)
for (phrase, canonical_index), dictionary_types in language_canonical_dictionaries.iteritems():
max_dictionary_types = max(max_dictionary_types, len(dictionary_types))
rule_template = address_expansion_rule_template.format(phrase=quote_string(phrase),
num_dictionaries=str(len(dictionary_types)),
dictionaries=', '.join(dictionary_types),
canonical_index=canonical_index)
expansion_rules.append(rule_template)
num_language_rules += 1
address_languages.append(address_language_index_template.format(language=quote_string(language),
index=language_index,
length=num_language_rules))
header = address_expansion_rule_header_template.format(
max_dictionary_types=str(max_dictionary_types)
)
out = open(header_file, 'w')
out.write(safe_encode(header))
out.close()
data_file = address_expansion_data_file_template.format(
canonical_strings=u''',
'''.join(canonical_strings),
expansion_rules=u''',
'''.join(expansion_rules),
address_languages=u''',
'''.join(address_languages),
)
out = open(output_file, 'w')
out.write(safe_encode(data_file))
out.close()
def create_intersections(self, outfile):
out = open(outfile, 'w')
for node_id, node_props, ways in self.intersections():
d = {'id': safe_encode(node_id), 'node': node_props, 'ways': ways}
out.write(json.dumps(d) + six.u('\n'))
def abbreviate(gazetteer,
s,
language,
abbreviate_prob=0.3,
separate_prob=0.2,
add_period_hyphen_prob=0.3):
'''
Abbreviations
-------------
OSM discourages abbreviations, but to make our training data map better
to real-world input, we can safely replace the canonical phrase with an
abbreviated version and retain the meaning of the words
'''
raw_tokens = tokenize_raw(s)
s_utf8 = safe_encode(s)
tokens = [(safe_decode(s_utf8[o:o + l]), token_types.from_id(c))
for o, l, c in raw_tokens]
norm_tokens = [(t.lower() if c in token_types.WORD_TOKEN_TYPES else t, c)
for t, c in tokens]
n = len(tokens)
abbreviated = []
i = 0
def abbreviated_tokens(i,
tokens,
t,
c,
length,
data,
space_token=six.u(' ')):
data = [d.split(six.b('|')) for d in data]
# local copy
abbreviated = []
n = len(t)
# Append the original tokens with whitespace if there is any
if random.random() > abbreviate_prob or not any(
(int(is_canonical) and lang in (language, 'all')
for lang, dictionary, is_canonical, canonical in data)):
for j, (t_i, c_i) in enumerate(t):
abbreviated.append(tokens[i + j][0])
if j < n - 1:
abbreviated.append(space_token)
return abbreviated
for lang, dictionary, is_canonical, canonical in data:
if lang not in (language, 'all'):
continue
is_canonical = int(is_canonical)
is_stopword = dictionary == 'stopword'
is_prefix = dictionary.startswith('concatenated_prefixes')
is_suffix = dictionary.startswith('concatenated_suffixes')
is_separable = is_prefix or is_suffix and dictionary.endswith(
'_separable') and len(t[0][0]) > length
suffix = None
prefix = None
if not is_canonical:
continue
if not is_prefix and not is_suffix:
abbreviations = gazetteer.canonicals.get(
(canonical, lang, dictionary))
# TODO: maybe make this a Zipfian choice e.g. so "St" gets chosen most often for "Street"
# would require an audit of the dictionaries though so abbreviations are listed from
# left-to-right by frequency of usage
token = random.choice(
abbreviations) if abbreviations else canonical
token = recase_abbreviation(token,
tokens[i:i + len(t)],
space_token=space_token)
abbreviated.append(token)
break
elif is_prefix:
token = tokens[i][0]
prefix, token = token[:length], token[length:]
abbreviated.append(prefix)
if random.random() < separate_prob:
sub_tokens = tokenize(token)
if sub_tokens and sub_tokens[0][1] in (token_types.HYPHEN,
token_types.DASH):
token = six.u('').join((t for t, c in sub_tokens[1:]))
abbreviated.append(space_token)
if token.islower():
abbreviated.append(token.title())
else:
abbreviated.append(token)
abbreviated.append(space_token)
break
elif is_suffix:
token = tokens[i][0]
token, suffix = token[:-length], token[-length:]
concatenated_abbreviations = gazetteer.canonicals.get(
(canonical, lang, dictionary), [])
separated_abbreviations = []
phrase = gazetteer.trie.get(suffix.rstrip('.'))
suffix_data = [
safe_decode(d).split(six.u('|')) for d in (phrase or [])
]
for l, d, _, c in suffix_data:
if l == lang and c == canonical:
separated_abbreviations.extend(
gazetteer.canonicals.get((canonical, lang, d)))
separate = random.random() < separate_prob
if concatenated_abbreviations and not separate:
abbreviation = random.choice(concatenated_abbreviations)
elif separated_abbreviations:
abbreviation = random.choice(separated_abbreviations)
else:
abbreviation = canonical
if separate:
sub_tokens = tokenize(token)
if sub_tokens and sub_tokens[-1][1] in (token_types.HYPHEN,
token_types.DASH):
token = six.u('').join((t for t, c in sub_tokens[:-1]))
abbreviated.append(token)
if separate:
abbreviated.append(space_token)
if suffix.isupper():
abbreviated.append(abbreviation.upper())
elif separate:
abbreviated.append(abbreviation.title())
else:
abbreviated.append(abbreviation)
break
else:
for j, (t_i, c_i) in enumerate(t):
abbreviated.append(tokens[i + j][0])
if j < n - 1:
abbreviated.append(space_token)
return abbreviated
for t, c, length, data in gazetteer.filter(norm_tokens):
if c == token_types.PHRASE:
abbrev_tokens = abbreviated_tokens(i, tokens, t, c, length, data)
abbreviated.extend(abbrev_tokens)
if i + len(t) < n and raw_tokens[i + len(t)][0] > sum(
raw_tokens[i + len(t) - 1][:2]):
abbreviated.append(six.u(' '))
i += len(t)
else:
token = tokens[i][0]
if not non_breaking_dash_regex.search(token):
abbreviated.append(token)
else:
sub_tokens = tokenize(
non_breaking_dash_regex.sub(six.u(' '), token))
sub_tokens_norm = [
(t.lower() if c in token_types.WORD_TOKEN_TYPES else t, c)
for t, c in sub_tokens
]
sub_token_abbreviated = []
sub_i = 0
sub_n = len(sub_tokens)
for t, c, length, data in gazetteer.filter(sub_tokens_norm):
if c == token_types.PHRASE:
abbrev_tokens = abbreviated_tokens(
sub_i,
sub_tokens,
t,
c,
length,
data,
space_token=six.u('-'))
sub_token_abbreviated.extend(abbrev_tokens)
sub_i += len(t)
if sub_i < sub_n:
if abbrev_tokens and random.random(
) < add_period_hyphen_prob and not abbrev_tokens[
-1].endswith(
six.u('.')) and not abbrev_tokens[
-1].lower().endswith(
sub_tokens_norm[sub_i - 1][0]):
sub_token_abbreviated.append(six.u('.'))
sub_token_abbreviated.append(six.u('-'))
else:
sub_token_abbreviated.append(sub_tokens[sub_i][0])
sub_i += 1
if sub_i < sub_n:
sub_token_abbreviated.append(six.u('-'))
abbreviated.append(six.u('').join(sub_token_abbreviated))
if i < n - 1 and raw_tokens[i + 1][0] > sum(raw_tokens[i][:2]):
abbreviated.append(six.u(' '))
i += 1
return six.u('').join(abbreviated).strip()
def create_geonames_tsv(db, out_dir=DEFAULT_DATA_DIR):
'''
Writes geonames.tsv using the specified db to the specified data directory
'''
filename = os.path.join(out_dir, 'geonames.tsv')
temp_filename = filename + '.tmp'
f = open(temp_filename, 'w')
writer = csv.writer(f, 'tsv_no_quote')
init_languages()
init_country_names()
wiki_titles = get_wikipedia_titles(db)
logging.info('Fetched Wikipedia titles')
# Iterate over GeoNames boundary types from largest (country) to smallest (neighborhood)
for boundary_type, codes in geonames_admin_dictionaries.iteritems():
if boundary_type != boundary_types.COUNTRY:
predicate = 'where gn.feature_code in ({codes})'.format(
codes=','.join(['"{}"'.format(c) for c in codes])
)
else:
# The query for countries in GeoNames is somewhat non-trivial
predicate = 'where gn.geonames_id in (select geonames_id from countries)'
query = base_geonames_query.format(
predicate=predicate
)
cursor = db.execute(query)
i = 1
while True:
# Fetch rows in batches to save memory
batch = cursor.fetchmany(BATCH_SIZE)
if not batch:
break
rows = []
for row in batch:
row = list(row)
row[DUMMY_BOUNDARY_TYPE_INDEX] = boundary_type
language = row[LANGUAGE_INDEX]
country_code = row[COUNTRY_CODE_INDEX]
is_preferred = int(row[PREFERRED_INDEX] or 0)
is_historical = int(row[HISTORICAL_INDEX] or 0)
lang_spoken = get_country_languages(country_code.lower(), official=False).get(language, None)
lang_official = get_country_languages(country_code.lower()).get(language, None) == 1
null_language = not language.strip()
is_canonical = row[NAME_INDEX] == row[CANONICAL_NAME_INDEX]
alpha2_code = None
is_orig_name = False
if boundary_type == boundary_types.COUNTRY:
alpha2_code = row[COUNTRY_CODE_INDEX]
is_orig_name = row[NAME_INDEX] == row[CANONICAL_NAME_INDEX] and row[LANGUAGE_INDEX] == ''
# Set the canonical for countries to the local name, see country_official_name in country_names.py
country_canonical = country_localized_display_name(alpha2_code.lower())
if not country_canonical or not country_canonical.strip():
raise ValueError('Could not get local canonical name for country code={}'.format(alpha2_code))
row[CANONICAL_NAME_INDEX] = country_canonical
geonames_id = row[GEONAMES_ID_INDEX]
name = utf8_normalize(safe_decode(row[NAME_INDEX]))
# For non-postal codes, don't count
if name.isdigit():
continue
wikipedia_entries = wiki_titles.get(name.lower(), wiki_titles.get(normalize_name(name.lower()), {}))
row[NAME_INDEX] = name
if boundary_type == boundary_types.COUNTRY:
norm_name = normalize_name(name.lower())
for s, repl in saint_replacements:
if not wikipedia_entries:
wikipedia_entries = wiki_titles.get(norm_name.replace(s, repl), {})
wiki_row = []
have_wikipedia = geonames_id in wikipedia_entries
wiki_preferred = wikipedia_entries.get(geonames_id, 0)
'''
The following set of heuristics assigns a numerical value to a given name
alternative, such that in the case of ambiguous names, this value can be
used as part of the ranking function (as indeed it will be during sort).
The higher the value, the more likely the given entity resolution.
'''
if is_historical:
# Historical names, unlikely to be used
language_priority = 0
elif not null_language and language != 'abbr' and lang_spoken is None:
# Name of a place in language not widely spoken e.g. Japanese name for a US toponym
language_priority = 1
elif null_language and not is_preferred and not is_canonical:
# Null-language alternate names not marked as preferred, dubious
language_priority = 2
elif language == 'abbr' and not is_preferred:
# Abbreviation, not preferred
language_priority = 3
elif language == 'abbr' and is_preferred:
# Abbreviation, preferred e.g. NYC, UAE
language_priority = 4
elif lang_spoken and not lang_official and not is_preferred:
# Non-preferred name but in a spoken (non-official) language
language_priority = 5
elif lang_official == 1 and not is_preferred:
# Name in an official language, not preferred
language_priority = 6
elif null_language and not is_preferred and is_canonical:
# Canonical name, may be overly official e.g. Islamic Republic of Pakistan
language_priority = 7
elif is_preferred and not lang_official:
# Preferred names, not an official language
language_priority = 8
elif is_preferred and lang_official:
# Official language preferred
language_priority = 9
row[DUMMY_LANGUAGE_PRIORITY_INDEX] = language_priority
if have_wikipedia:
wiki_row = row[:]
wiki_row[DUMMY_HAS_WIKIPEDIA_ENTRY_INDEX] = wiki_preferred + 1
rows.append(map(encode_field, wiki_row))
canonical = utf8_normalize(safe_decode(row[CANONICAL_NAME_INDEX]))
row[POPULATION_INDEX] = int(row[POPULATION_INDEX] or 0)
have_normalized = False
if is_orig_name:
canonical_row = wiki_row[:] if have_wikipedia else row[:]
canonical_row_name = normalize_display_name(name)
if canonical_row_name != name:
canonical_row[NAME_INDEX] = safe_encode(canonical_row_name)
have_normalized = True
rows.append(map(encode_field, canonical_row))
if not have_wikipedia:
rows.append(map(encode_field, row))
# Country names have more specialized logic
if boundary_type == boundary_types.COUNTRY:
wikipedia_entries = wiki_titles.get(canonical.lower(), {})
canonical_row_name = normalize_display_name(canonical)
canonical_row = row[:]
if is_orig_name:
canonical = safe_decode(canonical)
canonical_row[NAME_INDEX] = safe_encode(canonical)
norm_name = normalize_name(canonical.lower())
for s, repl in saint_replacements:
if not wikipedia_entries:
wikipedia_entries = wiki_titles.get(norm_name.replace(s, repl), {})
if not wikipedia_entries:
norm_name = normalize_name(canonical_row_name.lower())
for s, repl in saint_replacements:
if not wikipedia_entries:
wikipedia_entries = wiki_titles.get(norm_name.replace(s, repl), {})
have_wikipedia = geonames_id in wikipedia_entries
wiki_preferred = wikipedia_entries.get(geonames_id, 0)
if have_wikipedia:
canonical_row[DUMMY_HAS_WIKIPEDIA_ENTRY_INDEX] = wiki_preferred + 1
if (name != canonical):
rows.append(map(encode_field, canonical_row))
if canonical_row_name != canonical and canonical_row_name != name:
canonical_row[NAME_INDEX] = safe_encode(canonical_row_name)
rows.append(map(encode_field, canonical_row))
if alpha2_code and is_orig_name:
alpha2_row = row[:]
alpha2_row[NAME_INDEX] = alpha2_code
alpha2_row[DUMMY_LANGUAGE_PRIORITY_INDEX] = 10
rows.append(map(encode_field, alpha2_row))
if alpha2_code.lower() in country_alpha3_map and is_orig_name:
alpha3_row = row[:]
alpha3_row[NAME_INDEX] = country_code_alpha3_map[alpha2_code.lower()]
alpha3_row[DUMMY_LANGUAGE_PRIORITY_INDEX] = 10
rows.append(map(encode_field, alpha3_row))
writer.writerows(rows)
logging.info('Did {} batches'.format(i))
i += 1
cursor.close()
f.flush()
f.close()
logging.info('Sorting...')
env = os.environ.copy()
env['LC_ALL'] = 'C'
command = ['sort', '-t\t', '-u', '--ignore-case',
'-k{0},{0}'.format(NAME_INDEX + 1),
# If there's a Wikipedia link to this name for the given id, sort first
'-k{0},{0}nr'.format(DUMMY_HAS_WIKIPEDIA_ENTRY_INDEX + 1),
# Language priority rules as above
'-k{0},{0}nr'.format(DUMMY_LANGUAGE_PRIORITY_INDEX + 1),
# Sort descending by population (basic proxy for relevance)
'-k{0},{0}nr'.format(POPULATION_INDEX + 1),
# group rows for the same geonames ID together
'-k{0},{0}'.format(GEONAMES_ID_INDEX + 1),
# preferred names come first within that grouping
'-k{0},{0}nr'.format(PREFERRED_INDEX + 1),
# since uniquing is done on the sort key, add language
'-k{0},{0}'.format(LANGUAGE_INDEX + 1),
'-o', filename, temp_filename]
p = subprocess.Popen(command, env=env)
return_code = p.wait()
if return_code != 0:
raise subprocess.CalledProcessError(return_code, command)
os.unlink(temp_filename)
def tokenize(s, whitespace=False):
u = safe_decode(s)
s = safe_encode(s)
return [(safe_decode(s[start:start + length]), token_types.from_id(token_type))
for start, length, token_type in _tokenize.tokenize(u, whitespace)]
def include_polygon(self, props):
return 'ISO3166-1:alpha2' in props or 'ISO3166-2' in props or (props.get('type', 'relation'), safe_encode(props.get('id', ''))) in osm_admin1_ids
def path_and_filename(cls, wof_id):
id_str = safe_encode(wof_id)
n = 3
parts = [id_str[i:i + n] for i in six.moves.xrange(0, len(id_str), n)]
filename = six.u('{}.geojson').format(wof_id)
return six.u('/').join(parts), filename
def create_address_expansion_rules_file(base_dir=ADDRESS_EXPANSIONS_DIR,
output_file=ADDRESS_DATA_FILE,
header_file=ADDRESS_HEADER_FILE):
address_languages = []
expansion_rules = []
canonical_strings = []
max_dictionary_types = 0
for language in os.listdir(base_dir):
language_dir = os.path.join(base_dir, language)
num_language_rules = 0
language_index = len(expansion_rules)
language_canonical_dictionaries = defaultdict(list)
canonical_indices = {}
for filename in os.listdir(language_dir):
dictionary_name = filename.rstrip('.txt').lower()
if '.' in dictionary_name:
raise InvalidAddressFileException(
u'Invalid extension for file {}/{}, must be .txt'.format(
language, filename))
if dictionary_name not in gazetteer_types:
raise InvalidAddressFileException(
u'Invalid filename for file {}/{}. Must be one of {{{}}}'.
format(language, filename, ', '.join(gazetteer_types)))
dictionary_type = gazetteer_types[dictionary_name]
f = open(os.path.join(language_dir, filename))
for i, line in enumerate(f):
line = safe_decode(line.rstrip())
if not line.strip():
continue
if u'}' in line:
raise InvalidAddressFileException(
u'Found }} in file: {}/{}, line {}'.format(
language, filename, i + 1))
phrases = line.split(u'|')
if sum((1 for p in phrases if len(p.strip()) == 0)) > 0:
raise InvalidAddressFileException(
u'Found blank synonym in: {}/{}, line {}'.format(
language, filename, i + 1))
canonical = phrases[0]
if len(phrases) > 1:
canonical_index = canonical_indices.get(canonical, None)
if canonical_index is None:
canonical_index = len(canonical_strings)
canonical_strings.append(quote_string(canonical))
canonical_indices[canonical] = canonical_index
else:
canonical_index = -1
for i, p in enumerate(phrases):
language_canonical_dictionaries[
p, canonical_index if i > 0 else -1].append(
dictionary_type)
for (
phrase, canonical_index
), dictionary_types in language_canonical_dictionaries.iteritems():
max_dictionary_types = max(max_dictionary_types,
len(dictionary_types))
rule_template = address_expansion_rule_template.format(
phrase=quote_string(phrase),
num_dictionaries=str(len(dictionary_types)),
dictionaries=', '.join(dictionary_types),
canonical_index=canonical_index)
expansion_rules.append(rule_template)
num_language_rules += 1
address_languages.append(
address_language_index_template.format(
language=quote_string(language),
index=language_index,
length=num_language_rules))
header = address_expansion_rule_header_template.format(
max_dictionary_types=str(max_dictionary_types))
out = open(header_file, 'w')
out.write(safe_encode(header))
out.close()
data_file = address_expansion_data_file_template.format(
canonical_strings=u''',
'''.join(canonical_strings),
expansion_rules=u''',
'''.join(expansion_rules),
address_languages=u''',
'''.join(address_languages),
)
out = open(output_file, 'w')
out.write(safe_encode(data_file))
out.close()
def intersections(self):
'''
Generator which yields tuples like:
(node_id, lat, lon, {way_id: way_props})
'''
i = 0
node_ids = array.array('l')
node_counts = array.array('i')
for element_id, props, deps in parse_osm(self.filename, dependencies=True):
props = {safe_decode(k): safe_decode(v) for k, v in six.iteritems(props)}
if element_id.startswith('node'):
node_id = long(element_id.split(':')[-1])
node_ids.append(node_id)
node_counts.append(0)
self.node_props.Put(safe_encode(node_id), json.dumps(props))
elif element_id.startswith('way'):
# Don't care about the ordering of the nodes, and want uniques e.g. for circular roads
deps = set(deps)
# Get node indices by binary search
for node_id in deps:
try:
node_index = self.binary_search(node_ids, node_id)
except ValueError:
continue
if node_index is None:
continue
node_counts[node_index] += 1
if i % 1000 == 0 and i > 0:
self.logger.info('doing {}s, at {}'.format(element_id.split(':')[0], i))
i += 1
for i, count in enumerate(node_counts):
if count > 1:
self.node_ids.append(node_ids[i])
del node_ids
del node_counts
i = 0
for element_id, props, deps in parse_osm(self.filename, dependencies=True):
if element_id.startswith('node'):
node_id = long(element_id.split(':')[-1])
node_index = self.binary_search(self.node_ids, node_id)
elif element_id.startswith('way'):
props = {safe_decode(k): safe_decode(v) for k, v in six.iteritems(props)}
way_id = long(element_id.split(':')[-1])
props['id'] = way_id
for node_id in deps:
node_index = self.binary_search(self.node_ids, node_id)
if node_index is not None:
self.intersection_edges_nodes.append(node_id)
self.intersection_edges_ways.append(way_id)
self.way_props.Put(safe_encode(way_id), json.dumps(props))
if i % 1000 == 0 and i > 0:
self.logger.info('second pass, doing {}s, at {}'.format(element_id.split(':')[0], i))
i += 1
i = 0
indices = numpy.argsort(self.intersection_edges_nodes)
self.intersection_edges_nodes = numpy.fromiter((self.intersection_edges_nodes[i] for i in indices), dtype=numpy.uint64)
self.intersection_edges_ways = numpy.fromiter((self.intersection_edges_ways[i] for i in indices), dtype=numpy.uint64)
del indices
idx = 0
# Need to make a copy here otherwise will change dictionary size during iteration
for node_id, g in groupby(self.intersection_edges_nodes):
group_len = sum((1 for j in g))
node_props = json.loads(self.node_props.Get(safe_encode(node_id)))
way_indices = self.intersection_edges_ways[idx:idx + group_len]
all_ways = [json.loads(self.way_props.Get(safe_encode(w))) for w in way_indices]
way_names = set()
ways = []
for way in all_ways:
if way['name'] in way_names:
continue
ways.append(way)
way_names.add(way['name'])
idx += group_len
if i % 1000 == 0 and i > 0:
self.logger.info('checking intersections, did {}'.format(i))
i += 1
if len(ways) > 1:
node_index = self.binary_search(self.node_ids, node_id)
yield self.node_ids[node_index], node_props, ways
def create_address_expansion_rules_file(base_dir=ADDRESS_EXPANSIONS_DIR, output_file=ADDRESS_DATA_FILE, header_file=ADDRESS_HEADER_FILE):
address_languages = []
expansion_rules = []
canonical_strings = []
max_dictionary_types = 0
for language in os.listdir(base_dir):
language_dir = os.path.join(base_dir, language)
num_language_rules = 0
language_index = len(expansion_rules)
language_canonical_dictionaries = defaultdict(list)
canonical_indices = {}
for filename in os.listdir(language_dir):
dictionary_name = filename.rstrip('.txt').lower()
if '.' in dictionary_name:
raise InvalidAddressFileException(u'Invalid extension for file {}/{}, must be .txt'.format(language, filename))
if dictionary_name not in gazetteer_types:
raise InvalidAddressFileException(u'Invalid filename for file {}/{}. Must be one of {{{}}}'.format(language, filename, ', '.join(gazetteer_types)))
dictionary_type = gazetteer_types[dictionary_name]
f = open(os.path.join(language_dir, filename))
for i, line in enumerate(f):
line = safe_decode(line.rstrip())
if not line.strip():
continue
if u'}' in line:
raise InvalidAddressFileException(u'Found }} in file: {}/{}, line {}'.format(language, filename, i+1))
phrases = line.split(u'|')
if sum((1 for p in phrases if len(p.strip()) == 0)) > 0:
raise InvalidAddressFileException(u'Found blank synonym in: {}/{}, line {}'.format(language, filename, i+1))
canonical = phrases[0]
if len(phrases) > 1:
canonical_index = canonical_indices.get(canonical, None)
if canonical_index is None:
canonical_index = len(canonical_strings)
canonical_strings.append(quote_string(canonical))
canonical_indices[canonical] = canonical_index
else:
canonical_index = -1
for p in phrases:
language_canonical_dictionaries[(p, canonical_index)].append(dictionary_type)
for (phrase, canonical_index), dictionary_types in language_canonical_dictionaries.iteritems():
max_dictionary_types = max(max_dictionary_types, len(dictionary_types))
rule_template = address_expansion_rule_template.format(phrase=quote_string(phrase),
num_dictionaries=str(len(dictionary_types)),
dictionaries=', '.join(dictionary_types),
canonical_index=canonical_index)
expansion_rules.append(rule_template)
num_language_rules += 1
address_languages.append(address_language_index_template.format(language=quote_string(language),
index=language_index,
length=num_language_rules))
header = address_expansion_rule_header_template.format(
max_dictionary_types=str(max_dictionary_types)
)
out = open(header_file, 'w')
out.write(safe_encode(header))
out.close()
data_file = address_expansion_data_file_template.format(
canonical_strings=u''',
'''.join(canonical_strings),
expansion_rules=u''',
'''.join(expansion_rules),
address_languages=u''',
'''.join(address_languages),
)
out = open(output_file, 'w')
out.write(safe_encode(data_file))
out.close()
def __init__(self, config_file=BOUNDARY_NAMES_CONFIG):
config = yaml.load(open(config_file))
default_names = nested_get(config, ('names', 'keys'))
name_keys, probs = alternative_probabilities(default_names)
self.name_keys = name_keys
self.name_key_probs = cdf(probs)
self.component_name_keys = {}
for component, component_config in six.iteritems(
nested_get(config, ('names', 'components'), default={})):
component_names = component_config.get('keys')
component_name_keys, component_probs = alternative_probabilities(
component_names)
self.component_name_keys[component] = (component_name_keys,
cdf(component_probs))
self.country_regex_replacements = defaultdict(list)
for props in nested_get(config, (
'names',
'regex_replacements',
),
default=[]):
country = props.get('country')
re_flags = re.I | re.UNICODE
if not props.get('case_insensitive', True):
re.flags ^= re.I
pattern = re.compile(props['pattern'], re_flags)
replace_group = props['replace_with_group']
replace_probability = props['replace_probability']
self.country_regex_replacements[country].append(
(pattern, replace_group, replace_probability))
self.country_regex_replacements = dict(self.country_regex_replacements)
self.prefixes = {}
self.prefix_regexes = {}
self.suffixes = {}
self.suffix_regexes = {}
for language, components in six.iteritems(
nested_get(config, ('names', 'prefixes', 'language'),
default={})):
for component, affixes in six.iteritems(components):
affix_values, probs = alternative_probabilities(affixes)
for val in affix_values:
if 'prefix' not in val:
raise AssertionError(
six.
u('Invalid prefix value for (language={}, component={}): {} '
).format(language, component, val))
prefix_regex = six.u('|').join([
six.u('(?:{} )').format(self._string_as_regex(v['prefix']))
if v.get('whitespace') else self._string_as_regex(
v['prefix']) for v in affix_values
])
self.prefix_regexes[(language, component)] = re.compile(
six.u('^{}').format(prefix_regex), re.I | re.U)
if not isclose(sum(probs), 1.0):
affix_values.append(None)
probs.append(1.0 - sum(probs))
affix_probs_cdf = cdf(probs)
self.prefixes[(language,
component)] = affix_values, affix_probs_cdf
for language, components in six.iteritems(
nested_get(config, ('names', 'suffixes', 'language'),
default={})):
for component, affixes in six.iteritems(components):
affix_values, probs = alternative_probabilities(affixes)
for val in affix_values:
if 'suffix' not in val:
raise AssertionError(
six.
u('Invalid suffix value for (language={}, component={}): {} '
).format(language, component, val))
suffix_regex = six.u('|').join([
six.u('(?: {})').format(self._string_as_regex(v['suffix']))
if v.get('whitespace') else self._string_as_regex(
v['suffix']) for v in affix_values
])
self.suffix_regexes[(language, component)] = re.compile(
six.u('{}$').format(suffix_regex), re.I | re.U)
if not isclose(sum(probs), 1.0):
affix_values.append(None)
probs.append(1.0 - sum(probs))
affix_probs_cdf = cdf(probs)
self.suffixes[(language,
component)] = affix_values, affix_probs_cdf
self.exceptions = {}
for props in nested_get(config, ('names', 'exceptions'), default=[]):
object_type = props['type']
object_id = safe_encode(props['id'])
keys = [props['default']]
probs = [props['probability']]
for alt in props.get('alternatives', []):
keys.append(alt['alternative'])
probs.append(alt['probability'])
probs = cdf(probs)
self.exceptions[(object_type, object_id)] = (keys, probs)
def tokenize(s, whitespace=False):
u = safe_decode(s)
s = safe_encode(s)
return [(safe_decode(s[start:start + length]),
token_types.from_id(token_type))
for start, length, token_type in _tokenize.tokenize(u, whitespace)]
def tokenize(s):
u = safe_decode(s)
s = safe_encode(s)
return [(safe_decode(s[start:start + length]),
token_types.from_id(token_type))
for start, length, token_type in _tokenize.tokenize(u)]
def dropout_components(self, components, boundaries=(), country=None, population=None, unambiguous_city=False):
containing_ids = set()
for boundary in boundaries:
object_type = boundary.get('type')
object_id = safe_encode(boundary.get('id', ''))
if not (object_type and object_id):
continue
containing_ids.add((object_type, object_id))
original_bitset = ComponentDependencies.component_bitset(components)
names = defaultdict(list)
admin_components = [c for c in components if c in self.ADMIN_COMPONENTS]
for c in admin_components:
names[components[c]].append(c)
same_name = set()
for c, v in six.iteritems(names):
if len(v) > 1:
same_name |= set(v)
new_components = components.copy()
city_replacements = set()
if AddressFormatter.CITY not in components:
city_replacements = self.city_replacements(country)
for component in admin_components:
include = self.include_component(component, containing_ids, country=country, population=population, unambiguous_city=unambiguous_city)
if not include and component not in city_replacements:
# Note: this check is for cities that have the same name as their admin
# areas e.g. Luxembourg, Luxembourg. In cases like this, if we were to drop
# city, we don't want to include country on its own. This should help the parser
# default to the city in ambiguous cases where only one component is specified.
if not (component == AddressFormatter.CITY and component in same_name):
new_components.pop(component, None)
else:
value = components[component]
for c in names[value]:
new_components.pop(c, None)
for component in self.ADMIN_COMPONENTS:
value = self.get_property(('components', component, 'value'), country=country, default=None)
if not value:
values, probs = self.cdf_cache.get((country, component), (None, None))
if values is None:
values = self.get_property(('components', component, 'values'), country=country, default=None)
if values is not None:
values, probs = zip(*[(v['value'], float(v['probability'])) for v in values])
probs = cdf(probs)
self.cdf_cache[(country, component)] = (values, probs)
if values is not None:
value = weighted_choice(values, probs)
if value is not None and component not in components and self.include_component(component, containing_ids, country=country, population=population, unambiguous_city=unambiguous_city):
new_components[component] = value
self.drop_invalid_components(new_components, country, original_bitset=original_bitset)
return new_components
