def test_initials_surname_forname(self):
name = 'P.A. (Pia) Dijkstra'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'P.A. (Pia)Dijkstra'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
def test_initials_surname_forname(self):
name = 'P.A. (Pia) Dijkstra'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'P.A. (Pia)Dijkstra'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
def test_surname_prefix(self):
name = 'van Dijkstra, P.A.(Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
name = 'Dijkstra van, P.A. (Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
name = 'van Dijkstra,P.A.(Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
def test_surname_prefix(self):
name = 'van Dijkstra, P.A.(Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
name = 'Dijkstra van, P.A. (Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
name = 'van Dijkstra,P.A.(Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.assertEqual(surname_prefix, 'van')
self.check_result(initials, surname)
def create_votes_individual(self, voting, stemmingen):
logger.info('BEGIN')
for stemming in stemmingen:
persoon = stemming.persoon
if persoon:
initials = persoon.initialen
surname = persoon.achternaam
forname = persoon.roepnaam
else:
logger.error('Persoon not found for stemming: ' + stemming.id)
surname_initials = stemming.json['AnnotatieActorNaam']
forname = ''
initials, surname, surname_prefix = parse_name_surname_initials(surname_initials)
parliament_member = ParliamentMember.find(surname=surname, initials=initials)
if not parliament_member:
logger.error('parliament member not found for vote: ' + str(stemming.id))
logger.error('creating vote with empty parliament member')
person_name = ' '.join([forname, surname, initials]).strip()
VoteIndividual.objects.create(
voting=voting,
person_name=person_name,
parliament_member=parliament_member,
number_of_seats=1,
decision=self.get_decision(stemming.soort),
details='',
is_mistake=stemming.vergissing if stemming.vergissing is not None else False
)
logger.info('END')
def test_surname_initials(self):
name = 'Dijkstra, P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra,P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra P.A. (Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra, (Pia) P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
def create_votes_individual(voting, votes):
logger.info('BEGIN')
for vote in votes:
initials, surname, surname_prefix = parse_name_surname_initials(
vote.parliament_member)
parliament_member = ParliamentMember.find(surname=surname,
initials=initials)
if not parliament_member:
logger.error('parliament member not found for vote: ' + str(vote))
logger.error('creating vote with empty parliament member')
if voting.kamerstuk:
logger.error('on kamerstuk: ' + str(voting.kamerstuk) +
', in dossier: ' +
str(voting.kamerstuk.document.dossier) +
', for name: ' + surname + ' ' + initials)
else:
logger.error('voting.kamerstuk does not exist')
VoteIndividual.objects.create(voting=voting,
person_name=vote.parliament_member,
parliament_member=parliament_member,
number_of_seats=vote.number_of_seats,
decision=get_decision(vote.decision),
details=vote.details,
is_mistake=vote.is_mistake)
logger.info('END')
def set_derived(self):
initials, surname, surname_prefix = parse_name_surname_initials(
self.person_name)
self.parliament_member = ParliamentMember.find(surname=surname,
initials=initials,
date=self.voting.date)
self.save()
def get_person(document: Document, tk_person: TKPersoon = None, name: str = None):
person = None
if not tk_person:
logger.warning('No document submitter found for document: {}'.format(document.document_id))
else:
persons = Person.objects.filter(tk_id=tk_person.id)
if len(persons):
person = persons[0]
if person is None:
logger.warning('No person found for tk_persoon: {} ({})'.format(tk_person.achternaam, tk_person.initialen))
if person is None:
if tk_person and tk_person.achternaam and tk_person.initialen:
surname = tk_person.achternaam
surname_prefix = tk_person.tussenvoegsel
initials = tk_person.initialen
else:
initials, surname, surname_prefix = parse_name_surname_initials(name)
person = Person.find_surname_initials(surname=surname, initials=initials)
if not person:
logger.warning('Cannot find person: {} ({}). Creating new person!'.format(surname, initials))
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
return person
def test_surname_initials(self):
name = 'Dijkstra, P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra,P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra P.A. (Pia)'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
name = 'Dijkstra, (Pia) P.A.'
initials, surname, surname_prefix = parse_name_surname_initials(name)
self.check_result(initials, surname)
def create_submitter(document, submitter, date):
# TODO: fix this ugly if else mess
has_initials = len(submitter.split('.')) > 1
initials = ''
if has_initials:
initials, surname, surname_prefix = parse_name_surname_initials(submitter)
else:
surname, surname_prefix = parse_surname_comma_surname_prefix(submitter)
if initials == 'C.S.': # this is an abbreviation used in old metadata to indicate 'and usual others'
initials = ''
person = Person.find_surname_initials(surname=surname, initials=initials)
if surname == 'JAN JACOB VAN DIJK': # 'Jan Jacob van Dijk' and ' Jasper van Dijk' have the same surname and initials, to solve this they have included the forename in the surname
person = Person.objects.filter(forename='Jan Jacob', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JASPER VAN DIJK':
person = Person.objects.filter(forename='Jasper', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JAN DE VRIES':
person = Person.objects.filter(forename='Jan', surname_prefix='de', surname='Vries', initials='J.M.')[0]
if not person:
active_persons = SubmitterFactory.get_active_persons(date)
persons_similar = active_persons.filter(surname__iexact=surname).exclude(initials='').exclude(forename='')
if persons_similar.count() == 1:
person = persons_similar[0]
if not person:
if surname == '' and initials == '':
persons = Person.objects.filter(surname='', initials='', forename='')
if persons:
person = persons[0]
if not person:
persons = Person.objects.filter(surname__iexact=surname, initials__iexact=initials)
if persons:
person = persons[0]
if not person:
logger.warning('Cannot find person: ' + str(surname) + ' ' + str(initials) + '. Creating new person!')
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
party_members = PartyMember.get_at_date(person, date)
party_slug = ''
if party_members:
party_slug = party_members[0].party.slug
submitter, created = Submitter.objects.get_or_create(person=person, document=document, party_slug=party_slug)
return submitter
def create_votes_individual(self, voting, stemmingen):
logger.info('BEGIN')
for stemming in stemmingen:
persoon = stemming.persoon
parliament_member = None
persons = Person.objects.filter(tk_id=persoon.id)
if persons:
person = persons[0]
members = ParliamentMember.objects.filter(person=person).order_by('-joined')
parliament_member = members[0] if members.exists() else None
person_name = ' '.join([person.forename, person.surname, person.initials]).strip()
# TODO BR: this is a fallback, remove or extract function and log
if parliament_member is None:
if persoon:
initials = persoon.initialen
surname = persoon.achternaam
forname = persoon.roepnaam
else:
logger.error('Persoon not found for stemming: {}'.format(stemming.id))
surname_initials = stemming.json['AnnotatieActorNaam']
forname = ''
initials, surname, surname_prefix = parse_name_surname_initials(surname_initials)
parliament_member = ParliamentMember.find(surname=surname, initials=initials)
if not parliament_member:
logger.error('parliament member not found for vote: {}'.format(stemming.id))
logger.error('creating vote with empty parliament member')
person_name = ' '.join([forname, surname, initials]).strip()
VoteIndividual.objects.create(
voting=voting,
person_name=person_name,
person_tk_id=persoon.id,
parliament_member=parliament_member,
number_of_seats=1,
decision=self.get_decision(stemming.soort),
details='',
is_mistake=stemming.vergissing if stemming.vergissing is not None else False
)
logger.info('END')
def create_submitter(document, submitter, date):
# TODO: fix this ugly if else mess
has_initials = len(submitter.split('.')) > 1
initials = ''
if has_initials:
initials, surname, surname_prefix = parse_name_surname_initials(submitter)
else:
surname, surname_prefix = parse_surname_comma_surname_prefix(submitter)
if initials == 'C.S.': # this is an abbreviation used in old metadata to indicate 'and usual others'
initials = ''
person = Person.find_surname_initials(surname=surname, initials=initials)
if surname == 'JAN JACOB VAN DIJK': # 'Jan Jacob van Dijk' and ' Jasper van Dijk' have the same surname and initials, to solve this they have included the forename in the surname
person = Person.objects.filter(forename='Jan Jacob', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JASPER VAN DIJK':
person = Person.objects.filter(forename='Jasper', surname_prefix='van', surname='Dijk', initials='J.J.')[0]
if surname == 'JAN DE VRIES':
person = Person.objects.filter(forename='Jan', surname_prefix='de', surname='Vries', initials='J.M.')[0]
if not person:
active_persons = get_active_persons(date)
persons_similar = active_persons.filter(surname__iexact=surname).exclude(initials='').exclude(
forename='')
if persons_similar.count() == 1:
person = persons_similar[0]
if not person:
if surname == '' and initials == '':
persons = Person.objects.filter(surname='', initials='', forename='')
if persons:
person = persons[0]
if not person:
persons = Person.objects.filter(surname__iexact=surname, initials__iexact=initials)
if persons:
person = persons[0]
if not person:
logger.warning('Cannot find person: ' + str(surname) + ' ' + str(initials) + '. Creating new person!')
person = Person.objects.create(surname=surname, surname_prefix=surname_prefix, initials=initials)
party_members = PartyMember.get_at_date(person, date)
party_slug = ''
if party_members:
party_slug = party_members[0].party.slug
return Submitter.objects.create(person=person, document=document, party_slug=party_slug)
def set_derived(self):
initials, surname, surname_prefix = parse_name_surname_initials(self.person_name)
self.parliament_member = ParliamentMember.find(surname=surname, initials=initials, date=self.voting.date)
self.save()
