def createCountriesRelations(countryURI, countryEntity):
client = Client()
newData = ""
for rel in WIKIDATA_COUNTRY_RELATIONS:
pred = client.get(rel[1])
if pred in countryEntity:
for element in countryEntity.getlist(pred):
elementlabel = element.label
print(elementlabel)
countriesURI = getCountryURIByName(str(elementlabel))
if (len(countriesURI) == 0):
print("Could not find country " + str(elementlabel) +
" in local database")
else:
country2URI = countriesURI[0]
print(country2URI)
newData += "<" + BASE_URL + "/country/" + countryURI + "> <" + WIKIDATA_BASEURL + rel[
1] + "> <" + country2URI + "> . \n"
newData += "<" + BASE_URL + "/country/" + countryURI + "> <" + BASE_URL + "/hasRelations>" + " \"True\" .\n"
print(newData)
update = """INSERT DATA{""" + newData + """}"""
ins = executeInsert(update)
if len(ins) != 0:
print(ins)
def __init__(self, wikidata_id, max_dist=5, logger=None):
self.client = Client()
self.done = set()
self.logger = logger
self.max_dist = max_dist
self.get_individu(wikidata_id)
logger.warning(len(self.done))
def api_explore_country(country_id=country_id):
client = Client()
entity = client.get(country_id, load=True)
print('entity:', entity)
print('entity type:', entity.type)
print('id:', entity.id)
attributes = entity.attributes
attribute_keys = attributes.keys()
print(len(attribute_keys), 'attributes')
print('attribute keys:', attribute_keys)
print('title:', attributes['title'])
print('id:', attributes['id'])
labels = attributes['labels']
print(len(labels), 'labels')
print('Italian label:', labels['it'])
descriptions = attributes['descriptions']
print(len(descriptions), 'descriptions')
print('Italian description:', descriptions['it'])
claims = attributes['claims']
claim_keys = claims.keys()
print(len(claim_keys), 'claim keys')
print('claim keys:', claim_keys)
memberships = claims[MEMBER_OF]
print(len(memberships), 'membership claims')
# print('memberships:', memberships)
group_ids = [
ship['mainsnak']['datavalue']['value']['id'] for ship in memberships
]
print('belongs to groups:', group_ids)
groups = []
for group_id in group_ids[:3]:
time.sleep(SLEEP_TIME)
group_entity = client.get(group_id, load=False)
groups.append(group_entity.attributes['title'])
print('group titles:', groups)
def get_synonyms(wikibase_item):
"""
get the entity aliases
"""
client = Client() # doctest: +SKIP
entity = client.get(wikibase_item, load=True) # get a wikidata.Entity object for Operation Protective Edge
return entity.data['aliases'] # data is a dict inside the Entity object, 'aliases' returns all aliases for all the languages which have at least one alias for the label
def get_property_from_code(wikidata_code: str, property_code: str):
"""
Note: this is very inefficient at getting multiple properties.
:param wikidata_code: Q9928
:param property_code: P856
:return:
"""
if not wikidata_code or not property_code:
return ""
# log.debug(f"Wikidata code: {wikidata_code}, property_code: {property_code}")
try:
client = Client()
# reqests without a wikidata code result in raise HTTPError(req.full_url, code, msg, hdrs, fp)
entity = client.get(wikidata_code, load=True)
return_value = str(entity.get(client.get(property_code), None))
if not return_value or return_value == "None":
log.debug(f"No {property_code} for wikidata code: {wikidata_code}")
return ""
return return_value
except HTTPError:
# No entity with ID Q15111448 was found... etc.
# perfectly possible. In that case, no website, and thus continue.
# As this is a network call, million errors can happen, but what ones are actually a mystery.
return ""
def triplet_for_id(_id):
client = Client()
entity = client.get(_id, load=True)
triplets = []
if 'descriptions' in entity.attributes:
if 'hu' in entity.attributes['descriptions']:
triplets.append((_id, 'T_description',
entity.attributes['descriptions']['hu']['value']))
if 'aliases' in entity.attributes:
if 'hu' in entity.attributes['aliases']:
for a in entity.attributes['aliases']['hu']:
triplets.append((_id, 'T_alias', a['value']))
for k, v in entity.attributes['claims'].items():
try:
for lv in v:
triplets.append((_id, k, lv['mainsnak']['datavalue']['value']))
except:
pass
return pd.DataFrame(triplets, columns=['source', 'edge', 'destination'])
def __init__(self, initiator, category_id, room_id, backend):
self.room_id = room_id
self.game = Game.get_game(room_id, category_id) or Game(
initiator, category_id, room_id, backend)
self.question_types = QuestionType.objects.filter(
category_id=category_id)
self.client = Client()
def page_title_to_political_party(self, title):
"""
:param wiki: a valid WikipediaPage object
:return: A string representing the political party or affiliation of the entity described in the wikipage, if
available. Otherwise, a string 'None' is returned.
"""
# Go through wikipedia json to get the id for wikidata
try:
resp = requests.get(url='https://en.wikipedia.org/w/api.php?action=query&format=json&prop=pageprops&titles=' + title)
except:
return None
data = resp.json()
page_data = data['query']['pages'][list(data['query']['pages'].keys())[0]]
try:
page_properties = page_data['pageprops']
item_id = page_properties['wikibase_item']
except KeyError:
return 'None found'
# With item id in tow, extract political affiliation
client = Client()
entity = client.get(item_id, load=True)
try:
party_entity = entity.getlist(client.get('P102'))[0]
return str(party_entity.label)
except:
return 'None found'
def get_wikidataQ(self):
if self.wikidataQ != None:
html = '''<hr> <br> <br> Source: <a href="https://wikidata.org/wiki/Q''' + str(
self.wikidataQ) + '''" >Wikidata entry</a> <br> <br> '''
html += ''' <div class="row" >'''
cl = Client()
ent = cl.get("Q" + str(self.wikidataQ), load=True)
# first col
html += ''' <div class"col-6"> <h5> ''' + str(
ent.description) + ''' </h5> '''
# image
if "P18" in ent.data["claims"]:
prop = cl.get("P18")
thing = ent[prop]
html += ''' <img style="width:400px;heigth:400px" src="''' + thing.image_url + '''" alt=""> '''
html += ''' </div> '''
# end first col
if "P225" in ent.data["claims"]:
prop = cl.get("P225")
thing = ent[prop]
html += ''' <div class="col-4" style="margin-left:40px" > <h5> Taxon name: ''' + thing + ''' </h5> </div> '''
html += ''' </div> '''
return html
return ""
def fx_quantity_with_unit() -> Quantity:
client = Client()
# From Q520
return Quantity(
amount=610.13,
lower_bound=610.12,
upper_bound=610.14,
unit=client.get(EntityId("Q828224")))
def GetATPID(wd):
# Returns ATP-ID by Wikidata-ID
from wikidata.client import Client
client = Client()
entity = client.get(wd, load=True)
# get WTA-ID
wtaid = entity[client.get('P536')]
return (atpid)
def GetITFID(wd):
# Returns ITF-ID by Wikidata-ID
from wikidata.client import Client
client = Client()
entity = client.get(wd, load=True)
# get ITF-ID
itfid = entity[client.get('P599')]
return (itfid)
def test_client_get(fx_client: Client):
entity = fx_client.get(EntityId('Q1299'))
assert isinstance(entity, Entity)
assert entity.data is None
assert entity.id == EntityId('Q1299')
entity2 = fx_client.get(EntityId('Q1299'), load=True)
assert entity2.data is not None
assert entity2 is entity
def getWikidataCountryEntity(label):
try:
wikidataCountry = getWikidataCountry(label)[0]["country"]
except:
return None
client = Client()
entity = client.get(getWikidataEntity(wikidataCountry))
return entity
def getWikidataLabel(uri, language):
client = Client()
entity = client.get(extractEntity(uri), load=True)
try:
label = entity.label[Locale(language)]
except KeyError:
label = ''
return label
def get_property(self, entity_id, property_id):
wiki_client = Client()
print(entity_id)
entity = wiki_client.get(entity_id, True)
print(entity)
property = wiki_client.get(property_id)
print(property)
return entity[property]
def findAliases(id):
result = []
if id == '':
return ''
else:
client = Client()
result = []
entity = client.get(id, load=True)
def test_client_get(fx_client: Client):
entity = fx_client.get(EntityId('Q1299'))
assert isinstance(entity, Entity)
assert entity.data is None
assert entity.id == EntityId('Q1299')
entity2 = fx_client.get(EntityId('Q1299'), load=True)
assert entity2.data is not None
assert entity2 is entity
entity3 = fx_client.get(EntityId('1299'), load=True) # http 400 error
assert entity3.state is EntityState.non_existent
def get_wikidata_desc(q_number):
client = Client()
try:
entity = client.get(q_number, load=True)
_result = entity.description
except:
_result = None
return _result
def main():
client = Client()
entity = client.get('Q20145', load=True)
name_prop = client.get('P734')
name = entity[name_prop].label
# all_data = entity.data
# for i in all_data:
# print(i)
data = {'id': entity.id, 'description': entity.description, 'name': name}
print(data)
def getVictimRegion(victims):
allCoords = []
for victim in victims:
allCoords.append( (victim.location.longitude,victim.location.latitude) )
regionCounts = {}
allLocations = rg.search(allCoords,mode=1)
for i in range (len(allLocations)):
location = allLocations[i]
victim = victims[i]
locationAddress = location['name']+", "+location['admin1']+", "+location['admin2']
if locationAddress in regionCounts:
regionCounts[locationAddress].reportCount += 1
else:
addressParts = locationAddress.split(", ")
population = -1
for part in addressParts:
if population>0:
break
wikipage = None
wikidataPage = None
try:
wikipage = wptools.page(part,silent=True).get_parse()
client = Client()
wikidataPage = client.get(wikipage.data['wikibase'],load=True)
population = wikidataPage.data['claims']['P1082'][0]['mainsnak']['datavalue']['value']['amount']
population = int(population[1:])
except:
pass
if population>0:
thisRegion = VictimRegion(victim.location,population)
thisRegionCount = RegionCount(thisRegion,1)
regionCounts[locationAddress] = thisRegionCount
else:
population = 1
longituteFloat = float(location['lon'])
latitudeFloat = float(location['lat'])
key = str(int(longituteFloat*4.0))+", "+str(int(latitudeFloat*4.0))
if key in regionCounts:
regionCounts[key].reportCount += 1
else:
thisRegion = VictimRegion(victim.location, population)
thisRegionCount = RegionCount(thisRegion, 1)
regionCounts[key] = thisRegionCount
finalRegions = []
for countedRegionName in regionCounts:
thisRegionCount = regionCounts[countedRegionName]
if thisRegionCount.reportCount / thisRegionCount.region.populationCount > 1.00: # TODO: threshold tbd
finalRegions.append(thisRegionCount.region)
return finalRegions
def get_description_for_wikidata(wikidata_url):
parts = urlparse(wikidata_url)
wd_id = parts.path.split("/")[-1]
c = Client()
entity = c.get(wd_id, load=True)
sitelinks = entity.attributes.get("sitelinks", {})
enwiki = sitelinks.get("enwiki", {})
title = enwiki.get("title")
return get_description_from_wikipedia(title)
def test_client_guess_entity_type(
fx_client_opener: urllib.request.OpenerDirector):
guess_client = Client(opener=fx_client_opener, entity_type_guess=True)
assert guess_client.guess_entity_type(EntityId('Q1299')) is EntityType.item
assert (guess_client.guess_entity_type(EntityId('P434')) is
EntityType.property)
assert guess_client.guess_entity_type(EntityId('NotApplicable')) is None
noguess_client = Client(opener=fx_client_opener, entity_type_guess=False)
assert noguess_client.guess_entity_type(EntityId('Q1299')) is None
assert noguess_client.guess_entity_type(EntityId('P434')) is None
assert noguess_client.guess_entity_type(EntityId('NotApplicable')) is None
def get_image_from_entity_dict(self):
"""
Returns url to the full image path
@see https://stackoverflow.com/a/34402875
:return:
"""
client = Client()
entity = client.get(self.entity_id, load=True)
image_prop = client.get("P18")
image = entity[image_prop]
return image.image_url
def get_props(qid: str, pids: list) -> typing.Dict[str, str]:
cl = Client()
entity = cl.get(qid)
d = dict()
for pid in pids:
prop = cl.get(pid)
if prop in entity:
d[pid] = entity[prop]
return d
def main():
url = 'https://query.wikidata.org/#%23Lexemes%20describing%20a%20color%0A%23%20Lexemes%20describing%20a%20color%0A%23%20By%20Vesihiisi%0A%0ASELECT%20%3Fl%20%3Flemma%20%3FlanguageLabel%20%20WHERE%20%7B%0A%20%20%3Fl%20a%20ontolex%3ALexicalEntry%3B%20dct%3Alanguage%20%3Flanguage%3B%20wikibase%3Alemma%20%3Flemma%20.%0A%20%20%3Fl%20wdt%3AP31%20wd%3AQ376431.%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%22%5BAUTO_LANGUAGE%5D%2Cen%22.%20%7D%0A%7D%0AORDER%20BY%20%3FlanguageLabel'
all_entity = (get_html(url))
print(all_entity)
client = Client()
entity = client.get('Q20145', load=True)
name_prop = client.get('P734')
name = entity[name_prop].label
data = {'id': entity.id,
'description': entity.description,
'name': name}
print(data)
def get_sitelinks_by_linktype(qid: str, linktypes: list) -> typing.Dict[str, str]:
cl = Client()
entity = cl.get(qid)
entity.load()
entity_data = entity.data
d = dict()
if 'sitelinks' in entity_data:
for linktype in linktypes:
if linktype in entity_data['sitelinks']:
d[linktype] = entity_data['sitelinks'][linktype]['url']
return d
def validate_wikidata_movie_people(wikidata_id, people_by_pid):
cl = Client()
wiki_movie = cl.get(wikidata_id)
facts = {}
for role, people in people_by_pid.items():
wiki_people = wiki_movie.getlist(cl.get(role))
for person in people:
for wiki_person in wiki_people:
if is_same_person(wiki_person, person):
facts[person] = wiki_person.id
return facts
def test_entity_equality(fx_client_opener: urllib.request.OpenerDirector,
fx_client: Client, fx_loaded_entity: Entity):
# When entity id and client are the same
assert fx_loaded_entity is fx_client.get(fx_loaded_entity.id)
assert fx_loaded_entity == fx_client.get(fx_loaded_entity.id)
assert hash(fx_loaded_entity) == hash(fx_client.get(fx_loaded_entity.id))
# When entity id differs
assert fx_loaded_entity is not fx_client.get(EntityId('Q1299'))
assert fx_loaded_entity != fx_client.get(EntityId('Q1299'))
assert hash(fx_loaded_entity) != hash(fx_client.get(EntityId('Q1299')))
# When client differs
client = Client(opener=fx_client_opener)
assert fx_loaded_entity is not client.get(fx_loaded_entity.id)
assert fx_loaded_entity != client.get(fx_loaded_entity.id)
assert hash(fx_loaded_entity) != hash(client.get(fx_loaded_entity.id))
def save_data_to_db(self):
self.logger.info('Starting to save data')
self.client = Client()
category = Category.objects.create(name=self.category_name)
props = [self.client.get(alias, load=True) for alias in self.aliases]
for obj in self.data:
try:
db_obj = Object.objects.create(category_id=category.id, **obj)
self.logger.info(f"Saved {obj['name']} to database")
wikidata_obj = self.client.get(obj['wikidata_id'], load=True)
self.logger.info("Aliases loaded")
self.save_aliases(wikidata_obj, props, db_obj)
except Exception as e:
self.logger.error(f'Can not save to db: {obj}. Error: {e}')
return category.id
def get_wikidata_object(ref):
wikidata_obj_array = []
for ref_obj in ref.reference_objects:
if 'object_type' in ref_obj:
if ref_obj['object_type'].lower() in ['wikidata object', 'wikidata']:
wikidata_obj_array.append(ref_obj['object'])
if wikidata_obj_array:
return wikidata_obj_array
for wikidata_id in [get_wikidata_accession(ref)]:
client = Client()
entity = client.get(wikidata_id, load=True)
gnomics.objects.reference.Reference.add_object(ref, obj = entity.attributes, object_type = "Wikidata Object")
wikidata_obj_array.append(entity.attributes)
return wikidata_obj_array
def get_wikidata_object(compound, user=None):
wikidata_obj_array = []
for com_obj in compound.compound_objects:
if 'object_type' in com_obj:
if com_obj['object_type'].lower() in ['wikidata object', 'wikidata']:
wikidata_obj_array.append(assay_obj['object'])
if wikidata_obj_array:
return wikidata_obj_array
for wikidata_id in get_wikidata_accession(compound):
client = Client()
entity = client.get(wikidata_id, load=True)
gnomics.objects.compound.Compound.add_object(compound, obj=entity.attributes, object_type="Wikidata Object")
wikidata_obj_array.append(entity.attributes)
return wikidata_obj_array
def get_wikidata_object(person):
wikidata_obj_array = []
for per_obj in person.person_objects:
if 'object_type' in per_obj:
if per_obj['object_type'].lower() == 'wikidata object' or per_obj['object_type'].lower() == 'wikidata':
wikidata_obj_array.append(per_obj['object'])
if wikidata_obj_array:
return wikidata_obj_array
for wikidata_id in get_wikidata_accession(person):
client = Client()
entity = client.get(wikidata_id, load=True)
gnomics.objects.person.Person.add_object(person, obj = entity.attributes, object_type = "Wikidata Object")
wikidata_obj_array.append(entity.attributes)
return wikidata_obj_array
def get_wikidata_object(gene):
wikidata_obj_array = []
for gene_obj in gene.gene_objects:
if 'object_type' in gene_obj:
if gene_obj['object_type'].lower() in ['wikidata object', 'wikidata']:
wikidata_obj_array.append(gene_obj['object'])
if wikidata_obj_array:
return wikidata_obj_array
for wikidata_id in get_wikidata_accession(gene):
client = Client()
entity = client.get(wikidata_id, load=True)
gnomics.objects.gene.Gene.add_object(gene, obj = entity.attributes, object_type = "Wikidata Object")
wikidata_obj_array.append(entity.attributes)
return wikidata_obj_array
def get_wikidata_object(symptom):
wikidata_obj_array = []
for symp_obj in symptom.symptom_objects:
if 'object_type' in symp_obj:
if symp_obj['object_type'].lower() in ['wikidata object', 'wikidata']:
wikidata_obj_array.append(symp_obj['object'])
if wikidata_obj_array:
return wikidata_obj_array
for wikidata_id in [get_wikidata_accession(symptom)]:
client = Client()
entity = client.get(wikidata_id, load=True)
gnomics.objects.symptom.Symptom.add_object(taxon, obj = entity.attributes, object_type = "Wikidata Object")
wikidata_obj_array.append(entity.attributes)
return wikidata_obj_array
