def test_search_all(self): item = wikidata.WikidataItem(self.wikidata_id_ft) positions = item.get_positions_held() self.assertEqual(len(positions), 4) item = wikidata.WikidataItem(self.wikidata_id_wa) positions = item.get_positions_held() self.assertEqual(len(positions), 1)
def create_parliament_member_from_wikidata_id(parliament, person_wikidata_id): wikidata_item = wikidata.WikidataItem(person_wikidata_id) person = get_or_create_person(person_wikidata_id, wikidata_item=wikidata_item, add_initials=True) logger.info(person) positions = wikidata_item.get_parliament_positions_held() members = [] for position in positions: parliament_member = ParliamentMember.objects.create( person=person, parliament=parliament, joined=position['start_time'], left=position['end_time']) logger.info(parliament_member) members.append(parliament_member) if position['part_of_id']: party_item = wikidata.WikidataItem(position['part_of_id']) parties = PoliticalParty.objects.filter( wikidata_id=position['part_of_id']) if parties: party = parties[0] else: party = PoliticalParty.find_party( party_item.get_label(language='nl')) if not party: party = create_party_wikidata( wikidata_id=position['part_of_id']) PartyMember.objects.create(person=person, party=party, joined=position['start_time'], left=position['end_time']) return members
def test_search_parliament_member(self): item = wikidata.WikidataItem(self.wikidata_id_ft) positions = item.get_parliament_positions_held() self.assertEqual(len(positions), 2) for position in positions: self.assertEqual(position['id'], wikidata.PARLIAMENT_MEMBER_DUTCH_ITEM_ID) item = wikidata.WikidataItem(self.wikidata_id_mr) positions = item.get_parliament_positions_held() self.assertEqual(len(positions), 4)
def get_government_members(government_wikidata_id, max_members=None): logger.info('BEGIN') language = 'nl' parts = wikidata.WikidataItem(government_wikidata_id).get_parts() members = [] for part in parts: member = {} member['position_name'] = '' # used for ministers without portfolio member['properties'] = [] member['wikidata_id'] = part['mainsnak']['datavalue']['value']['id'] member_item = wikidata.WikidataItem(member['wikidata_id']) member['wikipedia_url'] = member_item.get_wikipedia_url( language=language) member['name'] = member_item.get_label(language=language) member[ 'parlement_and_politiek_id'] = member_item.get_parlement_and_politiek_id( ) for prop_id in part['qualifiers']: prop = part['qualifiers'][prop_id][0] # print(json.dumps(prop, sort_keys=True, indent=4)) if prop['datatype'] == 'wikibase-item': item_id = prop['datavalue']['value']['id'] # print(item_id) item_label = wikidata.WikidataItem(item_id).get_label( language=language) item_label = item_label.lower() member['properties'].append(item_label) if 'ministerie' in item_label: member['ministry'] = item_label.replace( 'ministerie van', '').strip() elif 'minister voor' in item_label: member['position_name'] = item_label.replace( 'minister voor', '').strip() elif 'position' not in member: if 'viceminister' in item_label or 'vicepremier' in item_label: member['position'] = 'viceminister-president' elif 'minister-president' in item_label or 'premier' in item_label: member['position'] = 'minister-president' elif 'staatssecretaris' in item_label: member['position'] = 'staatssecretaris' elif 'minister zonder portefeuille' in item_label: member['position'] = 'minister zonder portefeuille' elif 'minister' in item_label: member['position'] = 'minister' if prop['property'] == 'P580': # start time member['start_date'] = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) if prop['property'] == 'P582': # end time member['end_date'] = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) logger.info(member) members.append(member) if max_members and len(members) >= max_members: break logger.info('END') return members
def test_get_political_party_memberships_ignore_local_parties(self): loes_ypma_wikidata_id = 'Q1194971' item = wikidata.WikidataItem(loes_ypma_wikidata_id) parties = item.get_political_party_memberships() local_parties = 0 for party in parties: if wikidata.WikidataItem(party['party_wikidata_id']).is_local_party(): local_parties += 1 self.assertEqual(2, len(parties)) self.assertEqual(1, local_parties)
def get_government_members(government_wikidata_id, max_members=None) -> List[GovernmentMemberData]: logger.info('BEGIN') language = 'nl' parts = wikidata.WikidataItem(government_wikidata_id).get_parts() members = [] for part in parts: member = GovernmentMemberData() member.wikidata_id = part['mainsnak']['datavalue']['value']['id'] member_item = wikidata.WikidataItem(member.wikidata_id) member.wikipedia_url = member_item.get_wikipedia_url(language=language) member.name = member_item.get_label(language=language) member.parlement_and_politiek_id = member_item.get_parlement_and_politiek_id( ) for prop_id in part['qualifiers']: prop = part['qualifiers'][prop_id][0] # print(json.dumps(prop, sort_keys=True, indent=4)) if prop['datatype'] == 'wikibase-item': item_id = prop['datavalue']['value']['id'] item_label = wikidata.WikidataItem(item_id).get_label( language=language) item_label = item_label.lower() member.properties.append(item_label) if 'ministerie' in item_label: member.ministry = item_label.replace('ministerie van', '').strip() elif 'minister voor' in item_label or 'minister van' in item_label: member.position_name = item_label.replace( 'minister voor', '').replace('minister van', '').strip() elif member.position is None: if 'viceminister' in item_label or 'vicepremier' in item_label: member.position = 'viceminister-president' elif 'minister-president' in item_label or 'premier' in item_label: member.position = 'minister-president' elif 'staatssecretaris' in item_label: member.position = 'staatssecretaris' elif 'minister zonder portefeuille' in item_label: member.position = 'minister zonder portefeuille' elif 'minister' in item_label: member.position = 'minister' if prop['property'] == 'P580': # start time member.start_date = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) if prop['property'] == 'P582': # end time member.end_date = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) logger.info(json.dumps(member.__dict__, sort_keys=True, default=str)) members.append(member) if max_members and len(members) >= max_members: break logger.info('END') return members
def create_government_member(part, language) -> GovernmentMemberData: member = GovernmentMemberData() member.wikidata_id = part['mainsnak']['datavalue']['value']['id'] member_item = wikidata.WikidataItem(member.wikidata_id) member.wikipedia_url = member_item.get_wikipedia_url(language=language) member.name = member_item.get_label(language=language) member.parlement_and_politiek_id = member_item.get_parlement_and_politiek_id( ) for prop_id in part['qualifiers']: prop = part['qualifiers'][prop_id][0] # print(json.dumps(prop, sort_keys=True, indent=2)) if prop['datatype'] == 'wikibase-item': item_id = prop['datavalue']['value']['id'] item = wikidata.WikidataItem(item_id) is_without_portfolio = item.is_subclass_of_minister_without_portfolio( ) if is_without_portfolio: member.position = 'minister zonder portefeuille' item_label = item.get_label(language=language) item_label = item_label.lower() member.properties.append(item_label) if 'ministerie' in item_label: member.ministry = item_label.replace('ministerie van', '').strip() elif 'nederlands minister voor' in item_label or 'nederlands minister van' in item_label: member.position_name = item_label.replace( 'nederlands minister voor', '').replace('nederlands minister van', '').strip() elif 'minister voor' in item_label or 'minister van' in item_label: member.position_name = item_label.replace( 'minister voor', '').replace('minister van', '').strip() if member.position is None: if 'viceminister' in item_label or 'vicepremier' in item_label: member.position = 'viceminister-president' elif 'minister-president' in item_label or 'premier' in item_label: member.position = 'minister-president' elif 'staatssecretaris' in item_label: member.position = 'staatssecretaris' elif 'minister zonder portefeuille' in item_label: member.position = 'minister zonder portefeuille' elif 'minister ' in item_label or 'nederlandse minister' in item_label or 'minister' == item_label: member.position = 'minister' if prop['property'] == 'P580': # start time member.start_date = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) if prop['property'] == 'P582': # end time member.end_date = wikidata.WikidataItem.get_date( prop['datavalue']['value']['time']) return member
def get_government(government_wikidata_id): item = wikidata.WikidataItem(government_wikidata_id) return { 'name': item.get_label(language='nl'), 'start_date': item.get_start_time() or item.get_inception(), 'end_date': item.get_end_time() or item.get_dissolved(), }
def get_or_create_person(wikidata_id, fullname='', wikidata_item=None, add_initials=False): persons = Person.objects.filter(wikidata_id=wikidata_id) if not wikidata_item: wikidata_item = wikidata.WikidataItem(wikidata_id) if persons.exists(): person = persons[0] else: if not fullname: fullname = wikidata_item.get_label(language='nl') forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) person = Person.objects.create(forename=forename, surname=surname, surname_prefix=surname_prefix, wikidata_id=wikidata_id) person.update_info(language='nl', wikidata_item=wikidata_item) person.save() if add_initials and person.parlement_and_politiek_id: person.initials = scraper.persons.get_initials( person.parlement_and_politiek_id) person.save() assert person.wikidata_id == wikidata_id party_members = PartyMember.objects.filter(person=person) if not party_members.exists(): create_party_members_for_person(person) return person
def test_search_group_houwers(self): wikidata_id = wikidata.search_political_party_id('Houwers', language='nl') self.assertEqual(wikidata_id, 'Q28044763') item = wikidata.WikidataItem(wikidata_id) label = item.get_label(language='nl') self.assertEqual(label, 'Lid-Houwers')
def create_votes_party(voting, votes): logger.info('BEGIN') for vote in votes: party = PoliticalParty.find_party(vote.party_name) if not party: wikidata_id = wikidata.search_political_party_id(vote.party_name, language='nl') name = vote.party_name if wikidata_id: item = wikidata.WikidataItem(wikidata_id) name = item.get_label('nl') if not name: logger.error('vote party has no name') assert False party = PoliticalParty.objects.create(name=name, name_short=vote.party_name, wikidata_id=wikidata_id) party.update_info(language='nl') if not vote.decision: logger.warning('vote has no decision, vote.details: ' + str(vote.details)) VoteParty.objects.create(voting=voting, party=party, party_name=vote.party_name, number_of_seats=vote.number_of_seats, decision=get_decision(vote.decision), details=vote.details, is_mistake=vote.is_mistake) logger.info('END')
def test_search_socialist_party(self): wikidata_id = wikidata.search_political_party_id( 'Socialistische Partij', language='nl') self.assertEqual(wikidata_id, 'Q849580') item = wikidata.WikidataItem(wikidata_id) label = item.get_label(language='nl') self.assertEqual(label, 'Socialistische Partij')
def test_search_groenlinks(self): wikidata_id = wikidata.search_political_party_id('GL', language='nl') self.assertIsNotNone(wikidata_id) self.assertGreaterEqual(wikidata_id, 'Q667680') item = wikidata.WikidataItem(wikidata_id) label = item.get_label(language='nl') self.assertEqual(label, 'GroenLinks')
def create_party_wikidata(wikidata_id): wikidata_party_item = wikidata.WikidataItem(wikidata_id) name = wikidata_party_item.get_label(language='nl') name_short = wikidata_party_item.get_short_name(language='nl') if not name_short: name_short = name party = create_party(name=name, name_short=name_short) return party
def test_chantal_nijkerken_de_haan(self): wikidata_id = 'Q19830701' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Chantal') self.assertEqual(surname, 'Nijkerken-de Haan')
def test_leendert_de_lange(self): wikidata_id = 'Q19839084' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Leendert') self.assertEqual(surname_prefix, 'de') self.assertEqual(surname, 'Lange')
def test_melanie_schultz_van_hagen(self): wikidata_id = 'Q435886' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Melanie') self.assertEqual(surname_prefix, '') self.assertEqual(surname, 'Schultz van Haegen')
def test_sjoerd_sjoerdsma(self): wikidata_id = 'Q516335' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Sjoerd') self.assertEqual(surname, 'Sjoerdsma') self.assertEqual(surname_prefix, '')
def test_sybrand_van_haersma_buma(self): wikidata_id = 'Q377266' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Sybrand') self.assertEqual(surname, 'Haersma Buma') self.assertEqual(surname_prefix, 'van')
def test_jan_peter_balkenende(self): wikidata_id = 'Q133386' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Jan Peter') self.assertEqual(surname, 'Balkenende') self.assertEqual(surname_prefix, '')
def test_jan_kees_de_jager(self): wikidata_id = 'Q1666631' wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Jan Kees') self.assertEqual(surname, 'Jager') self.assertEqual(surname_prefix, 'de')
def test_fatma_koser_kaya(self): wikidata_id = 'Q467610' # Fatma Koşer Kaya wikidata_item = wikidata.WikidataItem(wikidata_id) fullname = wikidata_item.get_label() forename, surname, surname_prefix = Person.get_name_parts( fullname, wikidata_item) self.assertEqual(forename, 'Fatma') self.assertEqual(surname, 'Koşer Kaya') self.assertEqual(surname_prefix, '')
def test_get_fraction(self): wikidata_id = 'Q2801440' # Martin van Rooijen, 50Plus item_50plus_id = 'Q27122891' item = wikidata.WikidataItem(wikidata_id) positions = item.get_positions_held() fraction_id = None for position in positions: if position['id'] == wikidata.PARLIAMENT_MEMBER_DUTCH_ITEM_ID: fraction_id = position['part_of_id'] self.assertEqual(fraction_id, item_50plus_id)
def create_party_wikidata(wikidata_id): wikidata_party_item = wikidata.WikidataItem(wikidata_id) name = wikidata_party_item.get_label(language='nl') name_short = wikidata_party_item.get_short_name(language='nl') if not name_short: name_short = name filter_fractie = TKFractie.create_filter() filter_fractie.filter_fractie(naam=name) tk_fracties = tkapi.TKApi.get_fracties(filter=filter_fractie) tk_fractie_id = tk_fracties[0].id if tk_fracties else None party = create_party(name=name, name_short=name_short, tk_id=tk_fractie_id) return party
def test_get_frans_timmermans(self): logger.info('BEGIN') wikidata_id = 'Q32681' # Frans Timmermans item = wikidata.WikidataItem(wikidata_id) fullname = item.get_label() self.assertEqual(fullname, 'Frans Timmermans') given_name = item.get_given_names()[0] self.assertEqual(given_name, 'Frans') birth_date = item.get_birth_date() self.assertEqual(birth_date, datetime.date(day=6, month=5, year=1961)) parlement_positions = item.get_parliament_positions_held() self.assertEqual(len(parlement_positions), 2) logger.info('END')
def get_government_members(government_wikidata_id, max_members=None) -> List[GovernmentMemberData]: logger.info('BEGIN') language = 'nl' parts = wikidata.WikidataItem(government_wikidata_id).get_parts() members = [] for part in parts: member = create_government_member(part, language) logger.info(member.name) logger.info(json.dumps(member.__dict__, sort_keys=True, default=str)) members.append(member) if max_members and len(members) >= max_members: break logger.info('END') return members
def create_party_members_for_person(person: Person): logger.info('BEGIN - person: {}'.format(person)) if not person.wikidata_id: logger.warning( 'could not update party member for person: {} because person has no wikidata id.' .format(person)) return wikidata_person_item = wikidata.WikidataItem(person.wikidata_id) memberships = wikidata_person_item.get_political_party_memberships() PartyMember.objects.filter(person=person).delete() for membership in memberships: wikidata_party = wikidata.WikidataItem(membership['party_wikidata_id']) if wikidata_party.is_local_party(): continue parties = PoliticalParty.objects.filter(wikidata_id=wikidata_party.id) if parties.exists(): party = parties[0] else: party = create_party_wikidata(membership['party_wikidata_id']) PartyMember.objects.create(person=person, party=party, joined=membership['start_date'], left=membership['end_date']) logger.info('END')
def update_info(self, language='nl'): """ update the party with wikidata info :param language: the language to search for in wikidata """ if not self.wikidata_id: self.wikidata_id = self.find_wikidata_id(language) if not self.wikidata_id: return wikidata_item = wikidata.WikidataItem(self.wikidata_id) self.official_website_url = wikidata_item.get_official_website() self.wikipedia_url = wikidata_item.get_wikipedia_url(language) logger.info(self.name + ' - id: ' + str(self.wikidata_id)) logo_filename = wikidata_item.get_logo_filename() self.founded = wikidata_item.get_inception() if logo_filename: self.wikimedia_logo_url = wikidata.WikidataItem.get_wikimedia_image_url(logo_filename) self.save()
def update_info(self, language='nl', wikidata_item=None): logger.info('get info from wikidata for ' + str(self.get_full_name())) if not self.wikidata_id: self.wikidata_id = self.find_wikidata_id(language) if not self.wikidata_id: logger.warning('no wikidata id found') return if not wikidata_item: wikidata_item = wikidata.WikidataItem(self.wikidata_id) birthdate = wikidata_item.get_birth_date() if birthdate: self.birthdate = birthdate self.wikipedia_url = wikidata_item.get_wikipedia_url(language) image_filename = wikidata_item.get_image_filename() if image_filename: self.wikimedia_image_name = image_filename self.wikimedia_image_url = wikidata.WikidataItem.get_wikimedia_image_url(self.wikimedia_image_name) self.parlement_and_politiek_id = wikidata_item.get_parlement_and_politiek_id() self.twitter_username = wikidata_item.get_twitter_username()
def get_or_create_person(wikidata_id, fullname='', wikidata_item=None, add_initials=False): if not wikidata_item: wikidata_item = wikidata.WikidataItem(wikidata_id) persons = Person.objects.filter(wikidata_id=wikidata_id) if persons.count() > 1: logger.warning( 'more than one person with same wikidata_id found, wikidata id: ' + str(wikidata_id)) if persons.count() == 1: person = persons[0] else: person = create_person(wikidata_id, fullname, wikidata_item, add_initials) party_members = PartyMember.objects.filter(person=person) if not party_members.exists(): create_party_members_for_person(person) return person