def test_search_all(self):
item = wikidata.WikidataItem(self.wikidata_id_ft)
positions = item.get_positions_held()
self.assertEqual(len(positions), 4)
item = wikidata.WikidataItem(self.wikidata_id_wa)
positions = item.get_positions_held()
self.assertEqual(len(positions), 1)
def create_parliament_member_from_wikidata_id(parliament, person_wikidata_id):
wikidata_item = wikidata.WikidataItem(person_wikidata_id)
person = get_or_create_person(person_wikidata_id,
wikidata_item=wikidata_item,
add_initials=True)
logger.info(person)
positions = wikidata_item.get_parliament_positions_held()
members = []
for position in positions:
parliament_member = ParliamentMember.objects.create(
person=person,
parliament=parliament,
joined=position['start_time'],
left=position['end_time'])
logger.info(parliament_member)
members.append(parliament_member)
if position['part_of_id']:
party_item = wikidata.WikidataItem(position['part_of_id'])
parties = PoliticalParty.objects.filter(
wikidata_id=position['part_of_id'])
if parties:
party = parties[0]
else:
party = PoliticalParty.find_party(
party_item.get_label(language='nl'))
if not party:
party = create_party_wikidata(
wikidata_id=position['part_of_id'])
PartyMember.objects.create(person=person,
party=party,
joined=position['start_time'],
left=position['end_time'])
return members
def test_search_parliament_member(self):
item = wikidata.WikidataItem(self.wikidata_id_ft)
positions = item.get_parliament_positions_held()
self.assertEqual(len(positions), 2)
for position in positions:
self.assertEqual(position['id'], wikidata.PARLIAMENT_MEMBER_DUTCH_ITEM_ID)
item = wikidata.WikidataItem(self.wikidata_id_mr)
positions = item.get_parliament_positions_held()
self.assertEqual(len(positions), 4)
def get_government_members(government_wikidata_id, max_members=None):
logger.info('BEGIN')
language = 'nl'
parts = wikidata.WikidataItem(government_wikidata_id).get_parts()
members = []
for part in parts:
member = {}
member['position_name'] = '' # used for ministers without portfolio
member['properties'] = []
member['wikidata_id'] = part['mainsnak']['datavalue']['value']['id']
member_item = wikidata.WikidataItem(member['wikidata_id'])
member['wikipedia_url'] = member_item.get_wikipedia_url(
language=language)
member['name'] = member_item.get_label(language=language)
member[
'parlement_and_politiek_id'] = member_item.get_parlement_and_politiek_id(
)
for prop_id in part['qualifiers']:
prop = part['qualifiers'][prop_id][0]
# print(json.dumps(prop, sort_keys=True, indent=4))
if prop['datatype'] == 'wikibase-item':
item_id = prop['datavalue']['value']['id']
# print(item_id)
item_label = wikidata.WikidataItem(item_id).get_label(
language=language)
item_label = item_label.lower()
member['properties'].append(item_label)
if 'ministerie' in item_label:
member['ministry'] = item_label.replace(
'ministerie van', '').strip()
elif 'minister voor' in item_label:
member['position_name'] = item_label.replace(
'minister voor', '').strip()
elif 'position' not in member:
if 'viceminister' in item_label or 'vicepremier' in item_label:
member['position'] = 'viceminister-president'
elif 'minister-president' in item_label or 'premier' in item_label:
member['position'] = 'minister-president'
elif 'staatssecretaris' in item_label:
member['position'] = 'staatssecretaris'
elif 'minister zonder portefeuille' in item_label:
member['position'] = 'minister zonder portefeuille'
elif 'minister' in item_label:
member['position'] = 'minister'
if prop['property'] == 'P580': # start time
member['start_date'] = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
if prop['property'] == 'P582': # end time
member['end_date'] = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
logger.info(member)
members.append(member)
if max_members and len(members) >= max_members:
break
logger.info('END')
return members
def test_get_political_party_memberships_ignore_local_parties(self):
loes_ypma_wikidata_id = 'Q1194971'
item = wikidata.WikidataItem(loes_ypma_wikidata_id)
parties = item.get_political_party_memberships()
local_parties = 0
for party in parties:
if wikidata.WikidataItem(party['party_wikidata_id']).is_local_party():
local_parties += 1
self.assertEqual(2, len(parties))
self.assertEqual(1, local_parties)
def get_government_members(government_wikidata_id,
max_members=None) -> List[GovernmentMemberData]:
logger.info('BEGIN')
language = 'nl'
parts = wikidata.WikidataItem(government_wikidata_id).get_parts()
members = []
for part in parts:
member = GovernmentMemberData()
member.wikidata_id = part['mainsnak']['datavalue']['value']['id']
member_item = wikidata.WikidataItem(member.wikidata_id)
member.wikipedia_url = member_item.get_wikipedia_url(language=language)
member.name = member_item.get_label(language=language)
member.parlement_and_politiek_id = member_item.get_parlement_and_politiek_id(
)
for prop_id in part['qualifiers']:
prop = part['qualifiers'][prop_id][0]
# print(json.dumps(prop, sort_keys=True, indent=4))
if prop['datatype'] == 'wikibase-item':
item_id = prop['datavalue']['value']['id']
item_label = wikidata.WikidataItem(item_id).get_label(
language=language)
item_label = item_label.lower()
member.properties.append(item_label)
if 'ministerie' in item_label:
member.ministry = item_label.replace('ministerie van',
'').strip()
elif 'minister voor' in item_label or 'minister van' in item_label:
member.position_name = item_label.replace(
'minister voor', '').replace('minister van',
'').strip()
elif member.position is None:
if 'viceminister' in item_label or 'vicepremier' in item_label:
member.position = 'viceminister-president'
elif 'minister-president' in item_label or 'premier' in item_label:
member.position = 'minister-president'
elif 'staatssecretaris' in item_label:
member.position = 'staatssecretaris'
elif 'minister zonder portefeuille' in item_label:
member.position = 'minister zonder portefeuille'
elif 'minister' in item_label:
member.position = 'minister'
if prop['property'] == 'P580': # start time
member.start_date = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
if prop['property'] == 'P582': # end time
member.end_date = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
logger.info(json.dumps(member.__dict__, sort_keys=True, default=str))
members.append(member)
if max_members and len(members) >= max_members:
break
logger.info('END')
return members
def create_government_member(part, language) -> GovernmentMemberData:
member = GovernmentMemberData()
member.wikidata_id = part['mainsnak']['datavalue']['value']['id']
member_item = wikidata.WikidataItem(member.wikidata_id)
member.wikipedia_url = member_item.get_wikipedia_url(language=language)
member.name = member_item.get_label(language=language)
member.parlement_and_politiek_id = member_item.get_parlement_and_politiek_id(
)
for prop_id in part['qualifiers']:
prop = part['qualifiers'][prop_id][0]
# print(json.dumps(prop, sort_keys=True, indent=2))
if prop['datatype'] == 'wikibase-item':
item_id = prop['datavalue']['value']['id']
item = wikidata.WikidataItem(item_id)
is_without_portfolio = item.is_subclass_of_minister_without_portfolio(
)
if is_without_portfolio:
member.position = 'minister zonder portefeuille'
item_label = item.get_label(language=language)
item_label = item_label.lower()
member.properties.append(item_label)
if 'ministerie' in item_label:
member.ministry = item_label.replace('ministerie van',
'').strip()
elif 'nederlands minister voor' in item_label or 'nederlands minister van' in item_label:
member.position_name = item_label.replace(
'nederlands minister voor',
'').replace('nederlands minister van', '').strip()
elif 'minister voor' in item_label or 'minister van' in item_label:
member.position_name = item_label.replace(
'minister voor', '').replace('minister van', '').strip()
if member.position is None:
if 'viceminister' in item_label or 'vicepremier' in item_label:
member.position = 'viceminister-president'
elif 'minister-president' in item_label or 'premier' in item_label:
member.position = 'minister-president'
elif 'staatssecretaris' in item_label:
member.position = 'staatssecretaris'
elif 'minister zonder portefeuille' in item_label:
member.position = 'minister zonder portefeuille'
elif 'minister ' in item_label or 'nederlandse minister' in item_label or 'minister' == item_label:
member.position = 'minister'
if prop['property'] == 'P580': # start time
member.start_date = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
if prop['property'] == 'P582': # end time
member.end_date = wikidata.WikidataItem.get_date(
prop['datavalue']['value']['time'])
return member
def get_government(government_wikidata_id):
item = wikidata.WikidataItem(government_wikidata_id)
return {
'name': item.get_label(language='nl'),
'start_date': item.get_start_time() or item.get_inception(),
'end_date': item.get_end_time() or item.get_dissolved(),
}
def get_or_create_person(wikidata_id,
fullname='',
wikidata_item=None,
add_initials=False):
persons = Person.objects.filter(wikidata_id=wikidata_id)
if not wikidata_item:
wikidata_item = wikidata.WikidataItem(wikidata_id)
if persons.exists():
person = persons[0]
else:
if not fullname:
fullname = wikidata_item.get_label(language='nl')
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
person = Person.objects.create(forename=forename,
surname=surname,
surname_prefix=surname_prefix,
wikidata_id=wikidata_id)
person.update_info(language='nl', wikidata_item=wikidata_item)
person.save()
if add_initials and person.parlement_and_politiek_id:
person.initials = scraper.persons.get_initials(
person.parlement_and_politiek_id)
person.save()
assert person.wikidata_id == wikidata_id
party_members = PartyMember.objects.filter(person=person)
if not party_members.exists():
create_party_members_for_person(person)
return person
def test_search_group_houwers(self):
wikidata_id = wikidata.search_political_party_id('Houwers',
language='nl')
self.assertEqual(wikidata_id, 'Q28044763')
item = wikidata.WikidataItem(wikidata_id)
label = item.get_label(language='nl')
self.assertEqual(label, 'Lid-Houwers')
def create_votes_party(voting, votes):
logger.info('BEGIN')
for vote in votes:
party = PoliticalParty.find_party(vote.party_name)
if not party:
wikidata_id = wikidata.search_political_party_id(vote.party_name,
language='nl')
name = vote.party_name
if wikidata_id:
item = wikidata.WikidataItem(wikidata_id)
name = item.get_label('nl')
if not name:
logger.error('vote party has no name')
assert False
party = PoliticalParty.objects.create(name=name,
name_short=vote.party_name,
wikidata_id=wikidata_id)
party.update_info(language='nl')
if not vote.decision:
logger.warning('vote has no decision, vote.details: ' +
str(vote.details))
VoteParty.objects.create(voting=voting,
party=party,
party_name=vote.party_name,
number_of_seats=vote.number_of_seats,
decision=get_decision(vote.decision),
details=vote.details,
is_mistake=vote.is_mistake)
logger.info('END')
def test_search_socialist_party(self):
wikidata_id = wikidata.search_political_party_id(
'Socialistische Partij', language='nl')
self.assertEqual(wikidata_id, 'Q849580')
item = wikidata.WikidataItem(wikidata_id)
label = item.get_label(language='nl')
self.assertEqual(label, 'Socialistische Partij')
def test_search_groenlinks(self):
wikidata_id = wikidata.search_political_party_id('GL', language='nl')
self.assertIsNotNone(wikidata_id)
self.assertGreaterEqual(wikidata_id, 'Q667680')
item = wikidata.WikidataItem(wikidata_id)
label = item.get_label(language='nl')
self.assertEqual(label, 'GroenLinks')
def create_party_wikidata(wikidata_id):
wikidata_party_item = wikidata.WikidataItem(wikidata_id)
name = wikidata_party_item.get_label(language='nl')
name_short = wikidata_party_item.get_short_name(language='nl')
if not name_short:
name_short = name
party = create_party(name=name, name_short=name_short)
return party
def test_chantal_nijkerken_de_haan(self):
wikidata_id = 'Q19830701'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Chantal')
self.assertEqual(surname, 'Nijkerken-de Haan')
def test_leendert_de_lange(self):
wikidata_id = 'Q19839084'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Leendert')
self.assertEqual(surname_prefix, 'de')
self.assertEqual(surname, 'Lange')
def test_melanie_schultz_van_hagen(self):
wikidata_id = 'Q435886'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Melanie')
self.assertEqual(surname_prefix, '')
self.assertEqual(surname, 'Schultz van Haegen')
def test_sjoerd_sjoerdsma(self):
wikidata_id = 'Q516335'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Sjoerd')
self.assertEqual(surname, 'Sjoerdsma')
self.assertEqual(surname_prefix, '')
def test_sybrand_van_haersma_buma(self):
wikidata_id = 'Q377266'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Sybrand')
self.assertEqual(surname, 'Haersma Buma')
self.assertEqual(surname_prefix, 'van')
def test_jan_peter_balkenende(self):
wikidata_id = 'Q133386'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Jan Peter')
self.assertEqual(surname, 'Balkenende')
self.assertEqual(surname_prefix, '')
def test_jan_kees_de_jager(self):
wikidata_id = 'Q1666631'
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Jan Kees')
self.assertEqual(surname, 'Jager')
self.assertEqual(surname_prefix, 'de')
def test_fatma_koser_kaya(self):
wikidata_id = 'Q467610' # Fatma Koşer Kaya
wikidata_item = wikidata.WikidataItem(wikidata_id)
fullname = wikidata_item.get_label()
forename, surname, surname_prefix = Person.get_name_parts(
fullname, wikidata_item)
self.assertEqual(forename, 'Fatma')
self.assertEqual(surname, 'Koşer Kaya')
self.assertEqual(surname_prefix, '')
def test_get_fraction(self):
wikidata_id = 'Q2801440' # Martin van Rooijen, 50Plus
item_50plus_id = 'Q27122891'
item = wikidata.WikidataItem(wikidata_id)
positions = item.get_positions_held()
fraction_id = None
for position in positions:
if position['id'] == wikidata.PARLIAMENT_MEMBER_DUTCH_ITEM_ID:
fraction_id = position['part_of_id']
self.assertEqual(fraction_id, item_50plus_id)
def create_party_wikidata(wikidata_id):
wikidata_party_item = wikidata.WikidataItem(wikidata_id)
name = wikidata_party_item.get_label(language='nl')
name_short = wikidata_party_item.get_short_name(language='nl')
if not name_short:
name_short = name
filter_fractie = TKFractie.create_filter()
filter_fractie.filter_fractie(naam=name)
tk_fracties = tkapi.TKApi.get_fracties(filter=filter_fractie)
tk_fractie_id = tk_fracties[0].id if tk_fracties else None
party = create_party(name=name, name_short=name_short, tk_id=tk_fractie_id)
return party
def test_get_frans_timmermans(self):
logger.info('BEGIN')
wikidata_id = 'Q32681' # Frans Timmermans
item = wikidata.WikidataItem(wikidata_id)
fullname = item.get_label()
self.assertEqual(fullname, 'Frans Timmermans')
given_name = item.get_given_names()[0]
self.assertEqual(given_name, 'Frans')
birth_date = item.get_birth_date()
self.assertEqual(birth_date, datetime.date(day=6, month=5, year=1961))
parlement_positions = item.get_parliament_positions_held()
self.assertEqual(len(parlement_positions), 2)
logger.info('END')
def get_government_members(government_wikidata_id,
max_members=None) -> List[GovernmentMemberData]:
logger.info('BEGIN')
language = 'nl'
parts = wikidata.WikidataItem(government_wikidata_id).get_parts()
members = []
for part in parts:
member = create_government_member(part, language)
logger.info(member.name)
logger.info(json.dumps(member.__dict__, sort_keys=True, default=str))
members.append(member)
if max_members and len(members) >= max_members:
break
logger.info('END')
return members
def create_party_members_for_person(person: Person):
logger.info('BEGIN - person: {}'.format(person))
if not person.wikidata_id:
logger.warning(
'could not update party member for person: {} because person has no wikidata id.'
.format(person))
return
wikidata_person_item = wikidata.WikidataItem(person.wikidata_id)
memberships = wikidata_person_item.get_political_party_memberships()
PartyMember.objects.filter(person=person).delete()
for membership in memberships:
wikidata_party = wikidata.WikidataItem(membership['party_wikidata_id'])
if wikidata_party.is_local_party():
continue
parties = PoliticalParty.objects.filter(wikidata_id=wikidata_party.id)
if parties.exists():
party = parties[0]
else:
party = create_party_wikidata(membership['party_wikidata_id'])
PartyMember.objects.create(person=person,
party=party,
joined=membership['start_date'],
left=membership['end_date'])
logger.info('END')
def update_info(self, language='nl'):
"""
update the party with wikidata info
:param language: the language to search for in wikidata
"""
if not self.wikidata_id:
self.wikidata_id = self.find_wikidata_id(language)
if not self.wikidata_id:
return
wikidata_item = wikidata.WikidataItem(self.wikidata_id)
self.official_website_url = wikidata_item.get_official_website()
self.wikipedia_url = wikidata_item.get_wikipedia_url(language)
logger.info(self.name + ' - id: ' + str(self.wikidata_id))
logo_filename = wikidata_item.get_logo_filename()
self.founded = wikidata_item.get_inception()
if logo_filename:
self.wikimedia_logo_url = wikidata.WikidataItem.get_wikimedia_image_url(logo_filename)
self.save()
def update_info(self, language='nl', wikidata_item=None):
logger.info('get info from wikidata for ' + str(self.get_full_name()))
if not self.wikidata_id:
self.wikidata_id = self.find_wikidata_id(language)
if not self.wikidata_id:
logger.warning('no wikidata id found')
return
if not wikidata_item:
wikidata_item = wikidata.WikidataItem(self.wikidata_id)
birthdate = wikidata_item.get_birth_date()
if birthdate:
self.birthdate = birthdate
self.wikipedia_url = wikidata_item.get_wikipedia_url(language)
image_filename = wikidata_item.get_image_filename()
if image_filename:
self.wikimedia_image_name = image_filename
self.wikimedia_image_url = wikidata.WikidataItem.get_wikimedia_image_url(self.wikimedia_image_name)
self.parlement_and_politiek_id = wikidata_item.get_parlement_and_politiek_id()
self.twitter_username = wikidata_item.get_twitter_username()
def get_or_create_person(wikidata_id,
fullname='',
wikidata_item=None,
add_initials=False):
if not wikidata_item:
wikidata_item = wikidata.WikidataItem(wikidata_id)
persons = Person.objects.filter(wikidata_id=wikidata_id)
if persons.count() > 1:
logger.warning(
'more than one person with same wikidata_id found, wikidata id: ' +
str(wikidata_id))
if persons.count() == 1:
person = persons[0]
else:
person = create_person(wikidata_id, fullname, wikidata_item,
add_initials)
party_members = PartyMember.objects.filter(person=person)
if not party_members.exists():
create_party_members_for_person(person)
return person
