def test_find_by_surname_initials(self):
p_found = Person.find_surname_initials('Balkenende', 'J.P.')
self.assertEqual(p_found, self.p1)
p_found = Person.find_surname_initials('Balkenende', 'J.')
self.assertEqual(p_found, self.p2)
p_found = Person.find_surname_initials('Balkenende', '')
self.assertEqual(p_found, None)
p_found = Person.find_surname_initials('van der Steur', 'J.P.')
self.assertEqual(p_found, None)
p_found = Person.find_surname_initials('Koşer Kaya', 'F.')
self.assertEqual(p_found, self.p5)
def test_find_by_surname_initials(self):
p_found = Person.find_surname_initials('Balkenende', 'J.P.')
self.assertEqual(p_found, self.p1)
p_found = Person.find_surname_initials('Balkenende', 'J.')
self.assertEqual(p_found, self.p2)
p_found = Person.find_surname_initials('Balkenende', '')
self.assertEqual(p_found, None)
p_found = Person.find_surname_initials('van der Steur', 'J.P.')
self.assertEqual(p_found, None)
p_found = Person.find_surname_initials('Koşer Kaya', 'F.')
self.assertEqual(p_found, self.p5)
def get_person(document: Document, tk_person: TKPersoon = None, name: str = None):
person = None
if not tk_person:
logger.warning('No document submitter found for document: {}'.format(document.document_id))
else:
persons = Person.objects.filter(tk_id=tk_person.id)
if len(persons):
person = persons[0]
if person is None:
logger.warning('No person found for tk_persoon: {} ({})'.format(tk_person.achternaam, tk_person.initialen))
if person is None:
if tk_person and tk_person.achternaam and tk_person.initialen:
surname = tk_person.achternaam
surname_prefix = tk_person.tussenvoegsel
initials = tk_person.initialen
else:
initials, surname, surname_prefix = parse_name_surname_initials(name)
person = Person.find_surname_initials(surname=surname, initials=initials)
if not person:
logger.warning('Cannot find person: {} ({}). Creating new person!'.format(surname, initials))
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
return person
def create_travels(max_items=None):
travels = TKApi.get_items(PersoonReis, max_items=max_items)
logger.info('{} travels found.'.format(len(travels)))
if len(travels) < 1000:
logger.error(
'Only {} travels found. This is unexpected. Skip re-creating travels.'
.format(len(travels)))
Travel.objects.all().delete()
TravelPersonPosition.objects.all().delete()
for travel in travels:
person = Person.find_surname_initials(travel.persoon.achternaam,
travel.persoon.initialen)
if person is None:
logger.warning('No person found for travel: {}'.format(travel.id))
continue
if travel.van is None or travel.tot_en_met is None:
logger.warning('No date found for travel: {}'.format(travel.id))
continue
Travel.objects.create(
person=person,
destination=travel.bestemming,
purpose=travel.doel,
paid_by=travel.betaald_door,
date_begin=travel.van,
date_end=travel.tot_en_met,
)
def find(surname, initials='', date=None):
person = Person.find_surname_initials(surname, initials)
if date:
members = ParliamentMember.find_at_date(person, date)
else:
members = ParliamentMember.objects.filter(person=person).order_by('-joined')
if members.exists():
return members[0]
logger.info('ParliamentMember not found for: {} ({})'.format(surname, initials))
return None
def test_individual_voting(self):
pijkstra = Person.find_surname_initials('Dijkstra', 'P.A.')
vote = VoteIndividual.objects.create(
voting=self.voting,
person_name='Dijkstra, P.A.',
person_tk_id=pijkstra.tk_id,
number_of_seats=1
)
vote.set_derived()
self.assertEqual(vote.parliament_member.person.surname, 'Dijkstra')
def find(surname, initials='', date=None):
person = Person.find_surname_initials(surname, initials)
if date:
members = ParliamentMember.find_at_date(person, date)
else:
members = ParliamentMember.objects.filter(person=person).order_by('-joined')
if members.exists():
return members[0]
logger.info('ParliamentMember not found for: ' + str(surname) + ' (' + initials + ')')
return None
def create_submitter(document, submitter, date):
# TODO: fix this ugly if else mess
has_initials = len(submitter.split('.')) > 1
initials = ''
if has_initials:
initials, surname, surname_prefix = parse_name_surname_initials(submitter)
else:
surname, surname_prefix = parse_surname_comma_surname_prefix(submitter)
if initials == 'C.S.': # this is an abbreviation used in old metadata to indicate 'and usual others'
initials = ''
person = Person.find_surname_initials(surname=surname, initials=initials)
if surname == 'JAN JACOB VAN DIJK': # 'Jan Jacob van Dijk' and ' Jasper van Dijk' have the same surname and initials, to solve this they have included the forename in the surname
person = Person.objects.filter(forename='Jan Jacob', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JASPER VAN DIJK':
person = Person.objects.filter(forename='Jasper', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JAN DE VRIES':
person = Person.objects.filter(forename='Jan', surname_prefix='de', surname='Vries', initials='J.M.')[0]
if not person:
active_persons = SubmitterFactory.get_active_persons(date)
persons_similar = active_persons.filter(surname__iexact=surname).exclude(initials='').exclude(forename='')
if persons_similar.count() == 1:
person = persons_similar[0]
if not person:
if surname == '' and initials == '':
persons = Person.objects.filter(surname='', initials='', forename='')
if persons:
person = persons[0]
if not person:
persons = Person.objects.filter(surname__iexact=surname, initials__iexact=initials)
if persons:
person = persons[0]
if not person:
logger.warning('Cannot find person: ' + str(surname) + ' ' + str(initials) + '. Creating new person!')
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
party_members = PartyMember.get_at_date(person, date)
party_slug = ''
if party_members:
party_slug = party_members[0].party.slug
submitter, created = Submitter.objects.get_or_create(person=person, document=document, party_slug=party_slug)
return submitter
def create_submitter(document, submitter, date):
# TODO: fix this ugly if else mess
has_initials = len(submitter.split('.')) > 1
initials = ''
if has_initials:
initials, surname, surname_prefix = parse_name_surname_initials(submitter)
else:
surname, surname_prefix = parse_surname_comma_surname_prefix(submitter)
if initials == 'C.S.': # this is an abbreviation used in old metadata to indicate 'and usual others'
initials = ''
person = Person.find_surname_initials(surname=surname, initials=initials)
if surname == 'JAN JACOB VAN DIJK': # 'Jan Jacob van Dijk' and ' Jasper van Dijk' have the same surname and initials, to solve this they have included the forename in the surname
person = Person.objects.filter(forename='Jan Jacob', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JASPER VAN DIJK':
person = Person.objects.filter(forename='Jasper', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JAN DE VRIES':
person = Person.objects.filter(forename='Jan', surname_prefix='de', surname='Vries', initials='J.M.')[0]
if not person:
active_persons = get_active_persons(date)
persons_similar = active_persons.filter(surname__iexact=surname).exclude(initials='').exclude(
forename='')
if persons_similar.count() == 1:
person = persons_similar[0]
if not person:
if surname == '' and initials == '':
persons = Person.objects.filter(surname='', initials='', forename='')
if persons:
person = persons[0]
if not person:
persons = Person.objects.filter(surname__iexact=surname, initials__iexact=initials)
if persons:
person = persons[0]
if not person:
logger.warning('Cannot find person: ' + str(surname) + ' ' + str(initials) + '. Creating new person!')
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
party_members = PartyMember.get_at_date(person, date)
party_slug = ''
if party_members:
party_slug = party_members[0].party.slug
return Submitter.objects.create(person=person, document=document, party_slug=party_slug)
def create_travels(max_items=None):
travels = Api.get_items(PersoonReis, max_items=max_items)
if len(travels) < 1000:
logger.error('Only {} gifts found. This is unexpected. Skip re-creating gifts.'.format(len(travels)))
Travel.objects.all().delete()
TravelPersonPosition.objects.all().delete()
for travel in travels:
person = Person.find_surname_initials(travel.persoon.achternaam, travel.persoon.initialen)
if person is None:
logger.warning('No person found for travel: {}'.format(travel.id))
continue
if travel.van is None or travel.tot_en_met is None:
logger.warning('No date found for travel: {}'.format(travel.id))
continue
Travel.objects.create(
person=person,
destination=travel.bestemming,
purpose=travel.doel,
paid_by=travel.betaald_door,
date_begin=travel.van,
date_end=travel.tot_en_met,
)
def create_gifts(max_items=None):
gifts = Api.get_items(PersoonGeschenk, max_items=max_items)
if len(gifts) < 1000:
logger.error('Only {} gifts found. This is unexpected. Skip re-creating gifts.'.format(len(gifts)))
Gift.objects.all().delete()
PersonPosition.objects.all().delete()
for gift in gifts:
value = find_gift_value(gift.omschrijving)
gift_type = find_gift_type(gift.omschrijving)
person = Person.find_surname_initials(gift.persoon.achternaam, gift.persoon.initialen)
if person is None:
logger.warning('No person found for gift: {}'.format(gift.id))
continue
if gift.datum is None:
logger.warning('No date found for gift: {}'.format(gift.id))
continue
Gift.objects.create(
person=person,
value_euro=value,
description=gift.omschrijving,
date=gift.datum,
type=gift_type,
)
def try_merge_persons(persons):
for person in persons:
logger.info('===============')
logger.info('person: {}'.format(person))
if not person.surname or not person.initials:
continue
person_best = Person.find_surname_initials(person.surname,
person.initials,
persons)
persons_delete = persons.exclude(id=person_best.id)
logger.info('best {}, delete: {}'.format(person_best,
persons_delete))
submitters = Submitter.objects.filter(person__in=persons_delete)
for submitter in submitters:
if Submitter.objects.filter(
person=person_best,
document=submitter.document).exists():
continue
submitter.person = person_best
submitter.update_submitter_party_slug()
submitter.save()
persons_delete.delete()
break
def create_gifts(max_items=None):
gifts = TKApi.get_items(PersoonGeschenk, max_items=max_items)
logger.info('{} gifts found.'.format(len(gifts)))
if len(gifts) < 1000:
logger.error('Only {} gifts found. This is unexpected. Skip re-creating gifts.'.format(len(gifts)))
Gift.objects.all().delete()
PersonPosition.objects.all().delete()
for gift in gifts:
value = find_gift_value(gift.omschrijving)
gift_type = find_gift_type(gift.omschrijving)
person = Person.find_surname_initials(gift.persoon.achternaam, gift.persoon.initialen)
if person is None:
logger.warning('No person found for gift: {}'.format(gift.id))
continue
if gift.datum is None:
logger.warning('No date found for gift: {}'.format(gift.id))
continue
Gift.objects.create(
person=person,
value_euro=value,
description=gift.omschrijving,
date=gift.datum,
type=gift_type,
)
def test_find_surname_multiple(self):
p_found = Person.find_surname_initials('Grapperhaus', 'F.B.J.')
self.assertEqual(p_found, self.p9)
def test_find_surname_multiple(self):
p_found = Person.find_surname_initials('Grapperhaus', 'F.B.J.')
self.assertEqual(p_found, self.p9)
