def test_find_item_matches_pub(monkeypatch):
osm_tags = {
'amenity': 'university',
'building': 'university',
'name': 'Castle House',
}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'The Castle Inn'
}
},
'sitelinks': {},
}
tags = ['building', 'amenity=pub']
item = Item(entity=test_entity, tags=tags)
def mock_run_sql(cur, sql, debug):
return [('polygon', -295355, None, osm_tags, 12.75)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 0
def test_match_operator_at_start_of_name(monkeypatch):
osm_tags = {
'highway': 'services',
'landuse': 'commercial',
'name': 'Welcome Break Gordano Services',
'operator': 'Welcome Break',
}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Gordano services'
},
},
'sitelinks': {}
}
tags = ['highway=services']
item = Item(entity=test_entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('polygon', 64002602, None, osm_tags, 0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 1
def test_railway_station_cafe_bad_match(monkeypatch):
osm_tags = {
'addr:city': 'Gillingham',
'addr:housename': 'Gillingham (Kent) Railway Station',
'addr:postcode': 'ME7 1XE',
'addr:street': 'Railway Street',
'amenity': 'cafe',
'building': 'yes',
'cuisine': 'coffee_shop',
'name': 'BeeZoo Coffee Shop',
}
entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': "Gillingham railway station"
},
},
'sitelinks': {
'commonswiki': {
'site': 'commonswiki',
'title': 'Category:Gillingham (Kent) railway station',
},
'enwiki': {
'site': 'enwiki',
'title': 'Gillingham railway station (Kent)',
},
'nlwiki': {
'site': 'nlwiki',
'title': 'Station Gillingham (Kent)',
},
'simplewiki': {
'site': 'simplewiki',
'title': 'Gillingham (Kent) railway station',
},
},
}
tags = ['building=train_station', 'railway=station', 'railway=halt']
item = Item(entity=entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('polygon', 1, None, osm_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 0
def test_prefer_tag_match_over_building_only_match(monkeypatch):
tags1 = {
'name': 'Shepperton',
'network': 'National Rail',
'railway': 'station',
}
tags2 = {'building': 'yes', 'name': 'Shepperton Station'}
entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Shepperton railway station'
},
'nl': {
'language': 'nl',
'value': 'station Shepperton'
},
},
'sitelinks': {},
}
tags = ['building=train_station', 'railway=station', 'building']
item = Item(entity=entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('point', 3397249904, None, tags1, 26.78),
('polygon', 246812406, None, tags2, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 1
c = candidates[0]
del c['name_match']
assert c == {
'osm_type': 'node',
'osm_id': 3397249904,
'name': None,
'tags': tags1,
'dist': 26.78,
'planet_table': 'point',
'src_id': 3397249904,
'geom': None,
'identifier_match': False,
'address_match': None,
'matching_tags': {'railway=station'}
}
def find_item_matches(monkeypatch, osm_tags, item):
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('node', 1, None, osm_tags, 0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
return matcher.find_item_matches(mock_db, item, 'prefix')
def test_find_item_matches_parking(monkeypatch):
osm_tags = {
'amenity': 'parking',
'building': 'yes',
'fee': 'yes',
'name': 'PlayhouseSquare Parking',
'operator': 'PlayhouseSquare',
'parking': 'multi-storey',
'supervised': 'yes',
}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Playhouse Square'
},
'de': {
'language': 'de',
'value': 'Playhouse Square'
},
},
'sitelinks': {
'commonswiki': {
'site': 'commonswiki',
'title': 'Category:Playhouse Square',
},
'enwiki': {
'site': 'enwiki',
'title': 'Playhouse Square',
}
},
}
tags = ['amenity=arts_centre', 'building']
item = Item(entity=test_entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('polygon', 116620439, None, osm_tags, 253.7)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 0
def test_cottage_church_bad_match(monkeypatch):
church_tags = {
'amenity': 'place_of_worship',
'building': 'yes',
'denomination': 'anglican',
'name': 'Saint Anne',
'religion': 'christian',
}
entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': "St Anne's Cottage"
},
},
'sitelinks': {}
}
cottage_item = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': "cottage"
},
},
'sitelinks': {}
}
isa = IsA(item_id=5783996, entity=cottage_item)
tags = ['building', 'building=yes']
item = Item(entity=entity, tags=tags, isa=[isa])
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('polygon', 111491387, None, church_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 0
def test_alcatraz_lighthouse(monkeypatch):
lighthouse_tags = {
'alt_name': 'United States Coast Guard Lighthouse',
'building': 'yes',
'man_made': 'lighthouse',
'name': 'Alcatraz Island Lighthouse',
'start_date': '1909',
'wikidata': 'Q4712967',
}
island_tags = {'name': 'Alcatraz Island', 'tourism': 'attraction'}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Alcatraz Island Light'
},
},
'sitelinks': {
'commonswiki': {
'site': 'commonswiki',
'title': 'Category:Alcatraz Island Lighthouse'
},
'enwiki': {
'site': 'enwiki',
'title': 'Alcatraz Island Light'
},
},
}
tags = ['tourism=attraction', 'building', 'man_made=lighthouse']
item = Item(entity=test_entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('point', 265562462, None, island_tags, 151),
('polygon', 99202294, None, lighthouse_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 2
def test_name_and_location_better_than_address_and_building(monkeypatch):
tower_tags = {'name': 'Reunion Tower', 'tourism': 'attraction'}
hotel_tags = {
'addr:housenumber': '300',
'addr:street': 'Reunion Boulevard',
'building': 'hotel'
}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Reunion Tower'
},
},
'sitelinks': {},
}
extract = (
'<p><b>Reunion Tower</b> is a 561 ft (171 m) observation tower and ' +
'one of the most recognizable landmarks in Dallas, Texas. Located at '
+
'300 Reunion Boulevard in the Reunion district of downtown Dallas.</p>'
)
tags = ['man_made=tower', 'building=tower', 'height']
item = Item(entity=test_entity, tags=tags, extract=extract)
def mock_run_sql(cur, sql, debug):
if sql.startswith('select * from'):
return [('polygon', 29191381, None, hotel_tags, 0)]
else:
return [('point', 600482843, None, tower_tags, 7)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 2
def test_find_item_matches(monkeypatch):
osm_tags = {
'height': '44.9',
'building': 'yes',
'addr:street': 'West 37th Street',
'nycdoitt:bin': '1087066',
'addr:postcode': '10018',
'addr:housenumber': '450',
}
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('polygon', 265273006, None, osm_tags, 0.0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
extract = '''<p>The <b>Baryshnikov Arts Center</b> (<b>BAC</b>) is a foundation and arts complex opened by Mikhail Baryshnikov in 2005 at 450 West 37th Street between Ninth and Tenth Avenues in the Hell's Kitchen neighborhood of Manhattan, New York City. The top three floors of the complex are occupied by the Baryshnikov Arts Center, which provides space and production facilities for dance, music, theater, film, and visual arts. The building also houses the Orchestra of St. Luke's DiMenna Center for Classical Music.</p>'''
item = Item(entity=entity, tags=['building'], extract=extract)
mock_db = MockDatabase()
expect = {
'osm_type': 'way',
'osm_id': 265273006,
'name': None,
'tags': osm_tags,
'dist': 0.0,
'planet_table': 'polygon',
'src_id': 265273006,
'geom': None,
'identifier_match': False,
'address_match': True,
'name_match': {},
'matching_tags': {'building'},
}
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 1
assert candidates[0] == expect
def test_castle_station_bad_match(monkeypatch):
osm_tags = {
'building': 'train_station',
'name': 'Holyhead',
'name:cy': 'Caergybi',
'railway': 'station',
}
entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Caer Gybi'
},
'cy': {
'language': 'cy',
'value': 'Caer Gybi (caer)'
},
},
'sitelinks': {}
}
tags = ['historic=castle', 'building']
item = Item(entity=entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('polygon', 158252670, None, osm_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 0
def test_lifeboat_station_church_bad_match(monkeypatch):
osm_tags = {
'amenity': 'place_of_worship',
'building': 'yes',
'denomination': 'anglican',
'name': "St Agnes'",
'religion': 'christian',
}
entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'St Agnes Lifeboat Station'
},
},
'sitelinks': {}
}
tags = [
'amenity=lifeboat_station', 'building', 'building=yes',
'emergency=lifeboat_station'
]
item = Item(entity=entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('polygon', 234155614, None, osm_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 0
def test_find_item_matches_mall(monkeypatch):
osm_tags = {
'landuse': 'retail',
'name': 'Oxmoor Mall',
}
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': 'Oxmoor Center'
},
},
'sitelinks': {
'enwiki': {
'site': 'enwiki',
'title': 'Oxmoor Center',
}
},
}
tags = ['landuse=retail']
item = Item(entity=test_entity, tags=tags)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('polygon', 59847542, None, osm_tags, 0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 1
def test_church_is_not_school(monkeypatch):
test_entity = {
'claims': {},
'labels': {
'en': {
'language': 'en',
'value': "St. Paul's Catholic Church"
},
},
'sitelinks': {},
}
tags = ['amenity=place_of_worship', 'religion=christian']
item = Item(entity=test_entity, tags=tags)
osm_tags = {
'name': "Saint Paul's Catholic School",
'height': '12',
'amenity': 'school',
'building': 'school',
'religion': 'christian',
'denomination': 'catholic',
}
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('polygon', 1, None, osm_tags, 0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 0
def do_reindex(place, force=False):
print(place.display_name)
existing = {item.item_id: item.tags for item in place.items}
all_tags = place.all_tags
place.load_items()
place.add_tags_to_items()
print('tags updated')
tag_change = False
for item in place.items:
old = existing.get(item.item_id)
if old and set(item.tags) == set(old):
continue
tag_change = True
print(item.qid, item.enwiki)
if old:
print(' old:', old)
print(' new:', item.tags)
if not force and not tag_change:
print('no change')
place.state = 'ready'
database.session.commit()
return
place.wbgetentities()
database.session.commit()
print(sorted(place.all_tags))
print(sorted(all_tags))
sleep(10)
tables = database.get_tables()
expect = [place.prefix + '_' + t for t in ('line', 'point', 'polygon')]
if not all(t in tables for t in expect) or place.all_tags != all_tags:
if not place.overpass_done:
oql = place.get_oql()
overpass_url = 'https://overpass-api.de/api/interpreter'
wait_for_slot()
print('running overpass query')
r = requests.post(overpass_url,
data=oql,
headers=user_agent_headers())
print('overpass done')
place.save_overpass(r.content)
place.state = 'postgis'
database.session.commit()
print('running osm2pgsql')
place.load_into_pgsql(capture_stderr=False)
place.state = 'osm2pgsql'
database.session.commit()
conn = database.session.bind.raw_connection()
cur = conn.cursor()
q = place.items.filter(Item.entity.isnot(None)).order_by(Item.item_id)
for item in q:
candidates = matcher.find_item_matches(cur,
item,
place.prefix,
debug=False)
for i in (candidates or []):
c = ItemCandidate.query.get(
(item.item_id, i['osm_id'], i['osm_type']))
if not c:
c = ItemCandidate(**i, item=item)
database.session.add(c)
print(len(candidates), item.label)
place.state = 'ready'
database.session.commit()
conn.close()
def test_church_pub_bad_match(monkeypatch):
osm_tags = {
'amenity': 'pub',
'building': 'commercial',
'name': 'The Broadwater',
}
entity = {
'claims': {
'P373': [{
'mainsnak': {
'datavalue': {
'value': 'Broadwater Church, West Sussex'
},
}
}]
},
'labels': {
'en': {
'language': 'en',
'value': "St. Mary's Church, Broadwater"
},
},
'sitelinks': {
'commonswiki': {
'site': 'commonswiki',
'title': 'Category:Broadwater Church, West Sussex',
},
'enwiki': {
'site': 'enwiki',
'title': "St Mary's Church, Broadwater",
}
},
}
tags = [
'religion=christian', 'building=yes', 'building',
'amenity=place_of_worship', 'building=shrine', 'building=temple',
'building=church'
]
item = Item(entity=entity, tags=tags)
def place_names():
return ['West Sussex']
monkeypatch.setattr(item, 'place_names', place_names)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [
('polygon', 91013361, None, osm_tags, 0),
]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix', debug=True)
assert len(candidates) == 0
def test_embassy_no_match(monkeypatch):
osm_tags1 = {
'name': 'Consulate General of Switzerland in San Francisco',
'amenity': 'embassy',
'country': 'CH',
'addr:city': 'San Francisco',
'addr:state': 'CA',
'addr:street': 'Montgomery Street',
'addr:postcode': '94104',
'addr:housenumber': '456',
}
osm_tags2 = {
'addr:housenumber': '456',
'addr:street': 'Montgomery Street',
'building': 'yes',
'building:levels': '22',
'height': '114',
'name': 'Consulate General of Switzerland in San Francisco',
}
en_label = 'Consulate General of Israel to the Pacific Northwest Region'
test_entity = {
'claims': {
'P969': [{
'mainsnak': {
'datavalue': {
'value': '456 Montgomery Street Suite #2100'
},
},
}],
'P17': [{
'mainsnak': {
'datavalue': {
'value': {
'id': 'Q30'
}
}
},
}],
'P137': [{
'mainsnak': {
'datavalue': {
'value': {
'id': 'Q801'
}
}
},
}],
},
'labels': {
'en': {
'language': 'en',
'value': en_label
}
},
'sitelinks': {},
}
extract = (
'<p>The <b>Consulate General of Israel to the Pacific Northwest ' +
'Region</b>, is one of Israel’s diplomatic missions abroad, located at '
+
'456 Montgomery Street Suite #2100 in San Francisco, California, United '
+ 'States of America in the Financial District.</p>')
tags = ['amenity=embassy']
item = Item(entity=test_entity, tags=tags, extract=extract)
def mock_run_sql(cur, sql, debug):
if not sql.startswith('select * from'):
return []
return [('point', 1, None, osm_tags1, 0),
('polygon', 2, None, osm_tags2, 0)]
monkeypatch.setattr(matcher, 'run_sql', mock_run_sql)
monkeypatch.setattr(matcher, 'current_app', MockApp)
mock_db = MockDatabase()
candidates = matcher.find_item_matches(mock_db, item, 'prefix')
assert len(candidates) == 0
