def get_data():
results = collections.OrderedDict()
response = send_wikidata_query(SARQL_REQUEST)
for unit in response['results']['bindings']:
name = unit['item']['value'].replace('http://www.wikidata.org/entity/', '')
unit = unit['symbol']['value']
if name not in results:
# ignore duplicate: always use the first one
results[name] = unit
return results
def fetch_wikidata(nominatim_json, user_langage):
"""Update nominatim_json using the result of an unique to wikidata
For result in nominatim_json:
If result['extratags']['wikidata'] or r['extratags']['wikidata link']:
Set result['wikidata'] to { 'image': ..., 'image_sign':..., 'image_symbal':... }
Set result['extratags']['wikipedia'] if not defined
Set result['extratags']['contact:website'] if not defined
"""
wikidata_ids = []
wd_to_results = {}
for result in nominatim_json:
e = result.get("extratags")
if e:
# ignore brand:wikidata
wd_id = e.get("wikidata", e.get("wikidata link"))
if wd_id and wd_id not in wikidata_ids:
wikidata_ids.append("wd:" + wd_id)
wd_to_results.setdefault(wd_id, []).append(result)
if wikidata_ids:
wikidata_ids_str = " ".join(wikidata_ids)
query = wikidata_image_sparql.replace(
'%WIKIDATA_IDS%', sparql_string_escape(wikidata_ids_str)).replace(
'%LANGUAGE%', sparql_string_escape(user_langage))
wikidata_json = send_wikidata_query(query)
for wd_result in wikidata_json.get('results', {}).get('bindings', {}):
wd_id = wd_result['item']['value'].replace(
'http://www.wikidata.org/entity/', '')
for result in wd_to_results.get(wd_id, []):
result['wikidata'] = {
'itemLabel':
wd_result['itemLabel']['value'],
'image':
get_wikipedia_image(
wd_result.get('image', {}).get('value')),
'image_sign':
get_wikipedia_image(
wd_result.get('sign', {}).get('value')),
'image_symbol':
get_wikipedia_image(
wd_result.get('symbol', {}).get('value')),
}
# overwrite wikipedia link
wikipedia_name = wd_result.get('wikipediaName',
{}).get('value')
if wikipedia_name:
result['extratags'][
'wikipedia'] = user_langage + ':' + wikipedia_name
# get website if not already defined
website = wd_result.get('website', {}).get('value')
if (website and not result['extratags'].get('contact:website')
and not result['extratags'].get('website')):
result['extratags']['contact:website'] = website
def get_tags():
results = collections.OrderedDict()
response = send_wikidata_query(SPARQL_TAGS_REQUEST)
for tag in response['results']['bindings']:
tag_names = tag['tag']['value'].split(':')[1].split('=')
if len(tag_names) == 2:
tag_category, tag_type = tag_names
else:
tag_category, tag_type = tag_names[0], ''
label = tag['itemLabel']['value'].lower()
lang = tag['itemLabel']['xml:lang']
if lang in LANGUAGES:
results.setdefault(tag_category,
{}).setdefault(tag_type,
{}).setdefault(lang, label)
return results
def fetch_wikidata_descriptions():
global IDS
result = send_wikidata_query(
SPARQL_DESCRIPTION.replace('%IDS%',
IDS).replace('%LANGUAGES_SPARQL%',
LANGUAGES_SPARQL))
if result is not None:
for binding in result['results']['bindings']:
wikidata_id = binding['item']['value'].replace(
'http://www.wikidata.org/entity/', '')
lang = binding['itemDescription']['xml:lang']
description = binding['itemDescription']['value']
if ' ' in description: # skip unique word description (like "website")
for engine_name in wd_to_engine_name[wikidata_id]:
update_description(engine_name, lang, description,
'wikidata')
def get_data():
def get_key(unit):
return unit['item']['value'].replace('http://www.wikidata.org/entity/',
'')
def get_value(unit):
return unit['symbol']['value']
result = send_wikidata_query(SARQL_REQUEST)
if result is not None:
# sort the unit by entity name
# so different fetchs keep the file unchanged.
list(result['results']['bindings']).sort(key=get_key)
return collections.OrderedDict([
(get_key(unit), get_value(unit))
for unit in result['results']['bindings']
])
def fetch_wikipedia_descriptions():
global IDS
result = send_wikidata_query(
SPARQL_WIKIPEDIA_ARTICLE.replace('%IDS%',
IDS).replace('%LANGUAGES_SPARQL%',
LANGUAGES_SPARQL))
if result is not None:
for binding in result['results']['bindings']:
wikidata_id = binding['item']['value'].replace(
'http://www.wikidata.org/entity/', '')
lang = binding['name']['xml:lang']
pageid = binding['name']['value']
description = get_wikipedia_summary(lang, pageid)
if description is not None and ' ' in description:
for engine_name in wd_to_engine_name[wikidata_id]:
update_description(engine_name, lang, description,
'wikipedia')
def get_keys():
results = get_preset_keys()
response = send_wikidata_query(SPARQL_KEYS_REQUEST)
for key in response['results']['bindings']:
keys = key['key']['value'].split(':')[1:]
if keys[0] == 'currency' and len(keys) > 1:
# special case in openstreetmap.py
continue
if keys[0] == 'contact' and len(keys) > 1:
# label for the key "contact.email" is "Email"
# whatever the language
r = results.setdefault('contact', {})
r[keys[1]] = {'*': {'en': keys[1]}}
continue
if tuple(keys) in PRESET_KEYS:
# skip presets (already set above)
continue
if get_key_rank(':'.join(keys)) is None\
and ':'.join(keys) not in VALUE_TO_LINK\
and tuple(keys) not in INCLUDED_KEYS:
# keep only keys that will be displayed by openstreetmap.py
continue
label = key['itemLabel']['value'].lower()
lang = key['itemLabel']['xml:lang']
r = results
for k in keys:
r = r.setdefault(k, {})
r = r.setdefault('*', {})
if lang in LANGUAGES:
r.setdefault(lang, label)
# special cases
results['delivery']['covid19']['*'].clear()
for k, v in results['delivery']['*'].items():
results['delivery']['covid19']['*'][k] = v + ' (COVID19)'
results['opening_hours']['covid19']['*'].clear()
for k, v in results['opening_hours']['*'].items():
results['opening_hours']['covid19']['*'][k] = v + ' (COVID19)'
return results
def wikidata_request_result_iterator(request):
result = send_wikidata_query(
request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
if result is not None:
for r in result['results']['bindings']:
yield r